Package lib :: Package cuckoo :: Package core :: Module startup
[hide private]
[frames] | no frames]

Source Code for Module lib.cuckoo.core.startup

  1  # Copyright (C) 2010-2013 Claudio Guarnieri. 
  2  # Copyright (C) 2014-2016 Cuckoo Foundation. 
  3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 
  4  # See the file 'docs/LICENSE' for copying permission. 
  5   
  6  import httplib 
  7  import os 
  8  import shutil 
  9  import sys 
 10  import json 
 11  import socket 
 12  import urllib 
 13  import urllib2 
 14  import logging 
 15  import logging.handlers 
 16   
 17  from distutils.version import LooseVersion 
 18   
 19  from lib.cuckoo.common.colors import red, green, yellow 
 20  from lib.cuckoo.common.config import Config 
 21  from lib.cuckoo.common.constants import CUCKOO_ROOT, CUCKOO_VERSION 
 22  from lib.cuckoo.common.exceptions import CuckooStartupError, CuckooDatabaseError 
 23  from lib.cuckoo.common.exceptions import CuckooOperationalError 
 24  from lib.cuckoo.common.utils import create_folders 
 25  from lib.cuckoo.core.database import Database, TASK_RUNNING 
 26  from lib.cuckoo.core.database import TASK_FAILED_ANALYSIS, TASK_PENDING 
 27  from lib.cuckoo.core.log import DatabaseHandler, ConsoleHandler, TaskHandler 
 28  from lib.cuckoo.core.plugins import import_plugin, import_package, list_plugins 
 29  from lib.cuckoo.core.rooter import rooter, vpns 
 30   
 31  try: 
 32      import pwd 
 33      HAVE_PWD = True 
 34  except ImportError: 
 35      HAVE_PWD = False 
 36   
 37  log = logging.getLogger() 
 38   
39 -def check_python_version():
40 """Checks if Python version is supported by Cuckoo. 41 @raise CuckooStartupError: if version is not supported. 42 """ 43 if sys.version_info[:2] != (2, 7): 44 raise CuckooStartupError("You are running an incompatible version " 45 "of Python, please use 2.7")
46 47
48 -def check_working_directory():
49 """Checks if working directories are ready. 50 @raise CuckooStartupError: if directories are not properly configured. 51 """ 52 if not os.path.exists(CUCKOO_ROOT): 53 raise CuckooStartupError("You specified a non-existing root " 54 "directory: {0}".format(CUCKOO_ROOT)) 55 56 cwd = os.path.join(os.getcwd(), "cuckoo.py") 57 if not os.path.exists(cwd): 58 raise CuckooStartupError("You are not running Cuckoo from it's " 59 "root directory")
60 61
62 -def check_configs():
63 """Checks if config files exist. 64 @raise CuckooStartupError: if config files do not exist. 65 """ 66 configs = ( 67 "auxiliary.conf", "avd.conf", "cuckoo.conf", "esx.conf", "kvm.conf", 68 "memory.conf", "physical.conf", "processing.conf", "qemu.conf", 69 "reporting.conf", "virtualbox.conf", "vmware.conf", "vpn.conf", 70 "vsphere.conf", "xenserver.conf", 71 ) 72 73 for config in [os.path.join(CUCKOO_ROOT, "conf", f) for f in configs]: 74 if not os.path.exists(config): 75 raise CuckooStartupError("Config file does not exist at " 76 "path: {0}".format(config)) 77 78 return True
79
80 -def create_structure():
81 """Creates Cuckoo directories.""" 82 folders = [ 83 "log", 84 "storage", 85 os.path.join("storage", "analyses"), 86 os.path.join("storage", "binaries"), 87 os.path.join("storage", "baseline"), 88 ] 89 90 try: 91 create_folders(root=CUCKOO_ROOT, folders=folders) 92 except CuckooOperationalError as e: 93 raise CuckooStartupError(e)
94
95 -def check_version():
96 """Checks version of Cuckoo.""" 97 cfg = Config() 98 99 if not cfg.cuckoo.version_check: 100 return 101 102 print(" Checking for updates...") 103 104 url = "http://api.cuckoosandbox.org/checkversion.php" 105 data = urllib.urlencode({"version": CUCKOO_VERSION}) 106 107 try: 108 request = urllib2.Request(url, data) 109 response = urllib2.urlopen(request) 110 except (urllib2.URLError, urllib2.HTTPError, httplib.BadStatusLine): 111 print(red(" Failed! ") + "Unable to establish connection.\n") 112 return 113 114 try: 115 response_data = json.loads(response.read()) 116 except ValueError: 117 print(red(" Failed! ") + "Invalid response.\n") 118 return 119 120 stable_version = response_data["current"] 121 122 if CUCKOO_VERSION.endswith("-dev"): 123 print(yellow(" You are running a development version! Current stable is {}.".format( 124 stable_version))) 125 else: 126 if LooseVersion(CUCKOO_VERSION) < LooseVersion(stable_version): 127 msg = "Cuckoo Sandbox version {} is available now.".format( 128 stable_version) 129 130 print(red(" Outdated! ") + msg) 131 else: 132 print(green(" Good! ") + "You have the latest version " 133 "available.\n")
134
135 -def init_logging():
136 """Initializes logging.""" 137 formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") 138 139 fh = logging.handlers.WatchedFileHandler(os.path.join(CUCKOO_ROOT, "log", "cuckoo.log")) 140 fh.setFormatter(formatter) 141 log.addHandler(fh) 142 143 ch = ConsoleHandler() 144 ch.setFormatter(formatter) 145 log.addHandler(ch) 146 147 dh = DatabaseHandler() 148 dh.setLevel(logging.ERROR) 149 log.addHandler(dh) 150 151 th = TaskHandler() 152 th.setFormatter(formatter) 153 log.addHandler(th) 154 155 log.setLevel(logging.INFO)
156
157 -def init_console_logging():
158 """Initializes logging only to console.""" 159 formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s") 160 161 ch = ConsoleHandler() 162 ch.setFormatter(formatter) 163 log.addHandler(ch) 164 165 log.setLevel(logging.INFO)
166
167 -def init_tasks():
168 """Check tasks and reschedule uncompleted ones.""" 169 db = Database() 170 cfg = Config() 171 172 log.debug("Checking for locked tasks..") 173 for task in db.list_tasks(status=TASK_RUNNING): 174 if cfg.cuckoo.reschedule: 175 task_id = db.reschedule(task.id) 176 log.info( 177 "Rescheduled task with ID %s and target %s: task #%s", 178 task.id, task.target, task_id 179 ) 180 else: 181 db.set_status(task.id, TASK_FAILED_ANALYSIS) 182 log.info("Updated running task ID {0} status to failed_analysis".format(task.id)) 183 184 log.debug("Checking for pending service tasks..") 185 for task in db.list_tasks(status=TASK_PENDING, category="service"): 186 db.set_status(task.id, TASK_FAILED_ANALYSIS)
187
188 -def delete_file(*rel_path):
189 filepath = os.path.join(CUCKOO_ROOT, *rel_path) 190 if not os.path.exists(filepath): 191 return 192 193 try: 194 os.unlink(filepath) 195 except Exception as e: 196 log.warning( 197 "Unable to remove old %s leftover file from before you updated " 198 "your Cuckoo setup to the latest version: %s.", 199 os.path.join(*rel_path), e 200 )
201
202 -def init_modules(machinery=True):
203 """Initializes plugins.""" 204 log.debug("Importing modules...") 205 206 # Import all auxiliary modules. 207 import modules.auxiliary 208 import_package(modules.auxiliary) 209 210 # Import all processing modules. 211 import modules.processing 212 import_package(modules.processing) 213 214 # Import all signatures. 215 import modules.signatures 216 import_package(modules.signatures) 217 218 delete_file("modules", "reporting", "maec40.pyc") 219 delete_file("modules", "reporting", "maec41.pyc") 220 delete_file("modules", "reporting", "mmdef.pyc") 221 222 # Import all reporting modules. 223 import modules.reporting 224 import_package(modules.reporting) 225 226 # Import machine manager. 227 if machinery: 228 import_plugin("modules.machinery." + Config().cuckoo.machinery) 229 230 for category, entries in list_plugins().items(): 231 log.debug("Imported \"%s\" modules:", category) 232 233 for entry in entries: 234 if entry == entries[-1]: 235 log.debug("\t `-- %s", entry.__name__) 236 else: 237 log.debug("\t |-- %s", entry.__name__)
238
239 -def init_yara():
240 """Generates index for yara signatures.""" 241 log.debug("Initializing Yara...") 242 243 # Generate root directory for yara rules. 244 yara_root = os.path.join(CUCKOO_ROOT, "data", "yara") 245 246 # We divide yara rules in three categories. 247 categories = ["binaries", "urls", "memory"] 248 generated = [] 249 # Loop through all categories. 250 for category in categories: 251 # Check if there is a directory for the given category. 252 category_root = os.path.join(yara_root, category) 253 if not os.path.exists(category_root): 254 continue 255 256 # Check if the directory contains any rules. 257 signatures = [] 258 for entry in os.listdir(category_root): 259 if entry.endswith(".yara") or entry.endswith(".yar"): 260 signatures.append(os.path.join(category_root, entry)) 261 262 if not signatures: 263 continue 264 265 # Generate path for the category's index file. 266 index_name = "index_{0}.yar".format(category) 267 index_path = os.path.join(yara_root, index_name) 268 269 # Create index file and populate it. 270 with open(index_path, "w") as index_handle: 271 for signature in signatures: 272 index_handle.write("include \"{0}\"\n".format(signature)) 273 274 generated.append(index_name) 275 276 for entry in generated: 277 if entry == generated[-1]: 278 log.debug("\t `-- %s", entry) 279 else: 280 log.debug("\t |-- %s", entry)
281
282 -def init_binaries():
283 """Inform the user about the need to periodically look for new analyzer 284 binaries. These include the Windows monitor etc.""" 285 dirpath = os.path.join(CUCKOO_ROOT, "data", "monitor", "latest") 286 287 # Checks whether the "latest" symlink is available as well as whether 288 # it points to an existing directory. 289 if not os.path.exists(dirpath): 290 raise CuckooStartupError( 291 "The binaries used for Windows analysis are updated regularly, " 292 "independently from the release line. It appears that you're " 293 "not up-to-date. This can happen when you've just installed " 294 "Cuckoo or when you've updated your Cuckoo version by pulling " 295 "the latest changes from our Git repository. In order to get " 296 "up-to-date, please run the following " 297 "command: `./utils/community.py -wafb monitor` or " 298 "`./utils/community.py -waf` if you'd also like to download " 299 "over 300 Cuckoo signatures." 300 ) 301 302 # If "latest" is a file and not a symbolic link, check if it's destination 303 # directory is available. 304 if os.path.isfile(dirpath): 305 monitor = os.path.basename(open(dirpath, "rb").read().strip()) 306 dirpath = os.path.join(CUCKOO_ROOT, "data", "monitor", monitor) 307 else: 308 dirpath = None 309 310 if dirpath and not os.path.isdir(dirpath): 311 raise CuckooStartupError( 312 "The binaries used for Windows analysis are updated regularly, " 313 "independently from the release line. It appears that you're " 314 "not up-to-date. This can happen when you've just installed " 315 "Cuckoo or when you've updated your Cuckoo version by pulling " 316 "the latest changes from our Git repository. In order to get " 317 "up-to-date, please run the following " 318 "command: `./utils/community.py -wafb monitor` or " 319 "`./utils/community.py -waf` if you'd also like to download " 320 "over 300 Cuckoo signatures." 321 )
322
323 -def init_rooter():
324 """If required, check whether the rooter is running and whether we can 325 connect to it.""" 326 cuckoo = Config() 327 328 # The default configuration doesn't require the rooter to be ran. 329 if not Config("vpn").vpn.enabled and cuckoo.routing.route == "none": 330 return 331 332 cuckoo = Config() 333 s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) 334 335 try: 336 s.connect(cuckoo.cuckoo.rooter) 337 except socket.error as e: 338 if e.strerror == "No such file or directory": 339 raise CuckooStartupError( 340 "The rooter is required but it is either not running or it " 341 "has been configured to a different Unix socket path. " 342 "(In order to disable the use of rooter, please set route " 343 "and internet to none in cuckoo.conf and enabled to no in " 344 "vpn.conf)." 345 ) 346 347 if e.strerror == "Connection refused": 348 raise CuckooStartupError( 349 "The rooter is required but we can't connect to it as the " 350 "rooter is not actually running. " 351 "(In order to disable the use of rooter, please set route " 352 "and internet to none in cuckoo.conf and enabled to no in " 353 "vpn.conf)." 354 ) 355 356 if e.strerror == "Permission denied": 357 raise CuckooStartupError( 358 "The rooter is required but we can't connect to it due to " 359 "incorrect permissions. Did you assign it the correct group? " 360 "(In order to disable the use of rooter, please set route " 361 "and internet to none in cuckoo.conf and enabled to no in " 362 "vpn.conf)." 363 ) 364 365 raise CuckooStartupError("Unknown rooter error: %s" % e) 366 367 # Do not forward any packets unless we have explicitly stated so. 368 rooter("forward_drop")
369
370 -def init_routing():
371 """Initialize and check whether the routing information is correct.""" 372 cuckoo = Config() 373 vpn = Config("vpn") 374 375 # Check whether all VPNs exist if configured and make their configuration 376 # available through the vpns variable. Also enable NAT on each interface. 377 if vpn.vpn.enabled: 378 for name in vpn.vpn.vpns.split(","): 379 name = name.strip() 380 if not name: 381 continue 382 383 if not hasattr(vpn, name): 384 raise CuckooStartupError( 385 "Could not find VPN configuration for %s" % name 386 ) 387 388 entry = vpn.get(name) 389 390 if not rooter("nic_available", entry.interface): 391 raise CuckooStartupError( 392 "The network interface that has been configured for " 393 "VPN %s is not available." % entry.name 394 ) 395 396 if not rooter("rt_available", entry.rt_table): 397 raise CuckooStartupError( 398 "The routing table that has been configured for " 399 "VPN %s is not available." % entry.name 400 ) 401 402 vpns[entry.name] = entry 403 404 # Disable & enable NAT on this network interface. Disable it just 405 # in case we still had the same rule from a previous run. 406 rooter("disable_nat", entry.interface) 407 rooter("enable_nat", entry.interface) 408 409 # Populate routing table with entries from main routing table. 410 if cuckoo.routing.auto_rt: 411 rooter("flush_rttable", entry.rt_table) 412 rooter("init_rttable", entry.rt_table, entry.interface) 413 414 # Check whether the default VPN exists if specified. 415 if cuckoo.routing.route not in ("none", "internet"): 416 if not vpn.vpn.enabled: 417 raise CuckooStartupError( 418 "A VPN has been configured as default routing interface for " 419 "VMs, but VPNs have not been enabled in vpn.conf" 420 ) 421 422 if cuckoo.routing.route not in vpns: 423 raise CuckooStartupError( 424 "The VPN defined as default routing target has not been " 425 "configured in vpn.conf." 426 ) 427 428 # Check whether the dirty line exists if it has been defined. 429 if cuckoo.routing.internet != "none": 430 if not rooter("nic_available", cuckoo.routing.internet): 431 raise CuckooStartupError( 432 "The network interface that has been configured as dirty " 433 "line is not available." 434 ) 435 436 if not rooter("rt_available", cuckoo.routing.rt_table): 437 raise CuckooStartupError( 438 "The routing table that has been configured for dirty " 439 "line interface is not available." 440 ) 441 442 # Disable & enable NAT on this network interface. Disable it just 443 # in case we still had the same rule from a previous run. 444 rooter("disable_nat", cuckoo.routing.internet) 445 rooter("enable_nat", cuckoo.routing.internet) 446 447 # Populate routing table with entries from main routing table. 448 if cuckoo.routing.auto_rt: 449 rooter("flush_rttable", cuckoo.routing.rt_table) 450 rooter("init_rttable", cuckoo.routing.rt_table, 451 cuckoo.routing.internet)
452
453 -def cuckoo_clean():
454 """Clean up cuckoo setup. 455 It deletes logs, all stored data from file system and configured 456 databases (SQL and MongoDB). 457 """ 458 # Init logging. 459 # This need to init a console logger handler, because the standard 460 # logger (init_logging()) logs to a file which will be deleted. 461 create_structure() 462 init_console_logging() 463 464 # Initialize the database connection. 465 try: 466 db = Database(schema_check=False) 467 except CuckooDatabaseError as e: 468 # If something is screwed due to incorrect database migrations or bad 469 # database SqlAlchemy would be unable to connect and operate. 470 log.warning("Error connecting to database: it is suggested to check " 471 "the connectivity, apply all migrations if needed or purge " 472 "it manually. Error description: %s", e) 473 else: 474 # Drop all tables. 475 db.drop() 476 477 # Check if MongoDB reporting is enabled and drop that if it is. 478 cfg = Config("reporting") 479 if cfg.mongodb and cfg.mongodb.enabled: 480 from pymongo import MongoClient 481 host = cfg.mongodb.get("host", "127.0.0.1") 482 port = cfg.mongodb.get("port", 27017) 483 mdb = cfg.mongodb.get("db", "cuckoo") 484 try: 485 conn = MongoClient(host, port) 486 conn.drop_database(mdb) 487 conn.close() 488 except: 489 log.warning("Unable to drop MongoDB database: %s", mdb) 490 491 # Paths to clean. 492 paths = [ 493 os.path.join(CUCKOO_ROOT, "db"), 494 os.path.join(CUCKOO_ROOT, "log"), 495 os.path.join(CUCKOO_ROOT, "storage"), 496 ] 497 498 # Delete various directories. 499 for path in paths: 500 if os.path.isdir(path): 501 try: 502 shutil.rmtree(path) 503 except (IOError, OSError) as e: 504 log.warning("Error removing directory %s: %s", path, e) 505 506 # Delete all compiled Python objects ("*.pyc"). 507 for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT): 508 for fname in filenames: 509 if not fname.endswith(".pyc"): 510 continue 511 512 # We don't want to delete the Android's Agent .pyc files (as we 513 # don't ship the original .py files and thus they're critical). 514 if "agent/android/python_agent" in dirpath.replace("\\", "/"): 515 continue 516 517 path = os.path.join(dirpath, fname) 518 519 try: 520 os.unlink(path) 521 except (IOError, OSError) as e: 522 log.warning("Error removing file %s: %s", path, e)
523
524 -def drop_privileges(username):
525 """Drops privileges to selected user. 526 @param username: drop privileges to this username 527 """ 528 if not HAVE_PWD: 529 sys.exit("Unable to import pwd required for dropping " 530 "privileges (`pip install pwd`)") 531 532 try: 533 user = pwd.getpwnam(username) 534 os.setgroups((user.pw_gid,)) 535 os.setgid(user.pw_gid) 536 os.setuid(user.pw_uid) 537 os.putenv("HOME", user.pw_dir) 538 except KeyError: 539 sys.exit("Invalid user specified to drop privileges to: %s" % user) 540 except OSError as e: 541 sys.exit("Failed to drop privileges to %s: %s" % (username, e))
542