Package lib :: Package cuckoo :: Package common :: Module utils
[hide private]
[frames] | no frames]

Source Code for Module lib.cuckoo.common.utils

  1  # Copyright (C) 2010-2013 Claudio Guarnieri. 
  2  # Copyright (C) 2014-2016 Cuckoo Foundation. 
  3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 
  4  # See the file 'docs/LICENSE' for copying permission. 
  5   
  6  import hashlib 
  7  import logging 
  8  import os 
  9  import sys 
 10  import shutil 
 11  import ntpath 
 12  import string 
 13  import tempfile 
 14  import xmlrpclib 
 15  import inspect 
 16  import platform 
 17  import threading 
 18  import json 
 19  import multiprocessing 
 20  import warnings 
 21   
 22  from cStringIO import StringIO 
 23  from datetime import datetime 
 24   
 25  from lib.cuckoo.common.exceptions import CuckooOperationalError 
 26  from lib.cuckoo.common.config import Config 
 27   
 28  from lib.cuckoo.common.constants import CUCKOO_ROOT, CUCKOO_VERSION 
 29  from lib.cuckoo.common.constants import GITHUB_URL, ISSUES_PAGE_URL 
 30   
 31  try: 
 32      import bs4 
 33      HAVE_BS4 = True 
 34  except ImportError: 
 35      HAVE_BS4 = False 
 36   
 37  try: 
 38      import chardet 
 39      HAVE_CHARDET = True 
 40  except ImportError: 
 41      HAVE_CHARDET = False 
 42   
 43  try: 
 44      import jsbeautifier 
 45      HAVE_JSBEAUTIFIER = True 
 46  except ImportError: 
 47      HAVE_JSBEAUTIFIER = False 
 48   
 49  log = logging.getLogger(__name__) 
 50   
51 -def create_folders(root=".", folders=[]):
52 """Create directories. 53 @param root: root path. 54 @param folders: folders list to be created. 55 @raise CuckooOperationalError: if fails to create folder. 56 """ 57 for folder in folders: 58 create_folder(root, folder)
59
60 -def create_folder(root=".", folder=None):
61 """Create directory. 62 @param root: root path. 63 @param folder: folder name to be created. 64 @raise CuckooOperationalError: if fails to create folder. 65 """ 66 folder_path = os.path.join(root, folder) 67 if folder and not os.path.isdir(folder_path): 68 try: 69 os.makedirs(folder_path) 70 except OSError: 71 raise CuckooOperationalError("Unable to create folder: %s" % 72 folder_path)
73
74 -def delete_folder(folder):
75 """Delete a folder and all its subdirectories. 76 @param folder: path to delete. 77 @raise CuckooOperationalError: if fails to delete folder. 78 """ 79 if os.path.exists(folder): 80 try: 81 shutil.rmtree(folder) 82 except OSError: 83 raise CuckooOperationalError("Unable to delete folder: " 84 "{0}".format(folder))
85 86 # Don't allow all characters in "string.printable", as newlines, carriage 87 # returns, tabs, \x0b, and \x0c may mess up reports. 88 PRINTABLE_CHARACTERS = \ 89 string.letters + string.digits + string.punctuation + " \t\r\n" 90
91 -def convert_char(c):
92 """Escapes characters. 93 @param c: dirty char. 94 @return: sanitized char. 95 """ 96 if c in PRINTABLE_CHARACTERS: 97 return c 98 else: 99 return "\\x%02x" % ord(c)
100
101 -def is_printable(s):
102 """ Test if a string is printable.""" 103 for c in s: 104 if c not in PRINTABLE_CHARACTERS: 105 return False 106 return True
107
108 -def convert_to_printable(s):
109 """Convert char to printable. 110 @param s: string. 111 @return: sanitized string. 112 """ 113 if is_printable(s): 114 return s 115 return "".join(convert_char(c) for c in s)
116
117 -def datetime_to_iso(timestamp):
118 """Parse a datatime string and returns a datetime in iso format. 119 @param timestamp: timestamp string 120 @return: ISO datetime 121 """ 122 return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").isoformat()
123
124 -def get_filename_from_path(path):
125 """Cross-platform filename extraction from path. 126 @param path: file path. 127 @return: filename. 128 """ 129 dirpath, filename = ntpath.split(path) 130 return filename if filename else ntpath.basename(dirpath)
131
132 -def store_temp_file(filedata, filename, path=None):
133 """Store a temporary file. 134 @param filedata: content of the original file. 135 @param filename: name of the original file. 136 @param path: optional path for temp directory. 137 @return: path to the temporary file. 138 """ 139 filename = get_filename_from_path(filename) 140 141 # Reduce length (100 is arbitrary). 142 filename = filename[:100] 143 144 options = Config() 145 # Create temporary directory path. 146 if path: 147 target_path = path 148 else: 149 tmp_path = options.cuckoo.get("tmppath", "/tmp") 150 target_path = os.path.join(tmp_path, "cuckoo-tmp") 151 if not os.path.exists(target_path): 152 os.mkdir(target_path) 153 154 tmp_dir = tempfile.mkdtemp(prefix="upload_", dir=target_path) 155 tmp_file_path = os.path.join(tmp_dir, filename) 156 with open(tmp_file_path, "wb") as tmp_file: 157 # If filedata is file object, do chunked copy. 158 if hasattr(filedata, "read"): 159 chunk = filedata.read(1024) 160 while chunk: 161 tmp_file.write(chunk) 162 chunk = filedata.read(1024) 163 else: 164 tmp_file.write(filedata) 165 166 return tmp_file_path
167
168 -class TimeoutServer(xmlrpclib.ServerProxy):
169 """Timeout server for XMLRPC. 170 XMLRPC + timeout - still a bit ugly - but at least gets rid of setdefaulttimeout 171 inspired by http://stackoverflow.com/questions/372365/set-timeout-for-xmlrpclib-serverproxy 172 (although their stuff was messy, this is cleaner) 173 @see: http://stackoverflow.com/questions/372365/set-timeout-for-xmlrpclib-serverproxy 174 """
175 - def __init__(self, *args, **kwargs):
176 timeout = kwargs.pop("timeout", None) 177 kwargs["transport"] = TimeoutTransport(timeout=timeout) 178 xmlrpclib.ServerProxy.__init__(self, *args, **kwargs)
179
180 - def _set_timeout(self, timeout):
181 t = self._ServerProxy__transport 182 t.timeout = timeout 183 # If we still have a socket we need to update that as well. 184 if hasattr(t, "_connection") and t._connection[1] and t._connection[1].sock: 185 t._connection[1].sock.settimeout(timeout)
186
187 -class TimeoutTransport(xmlrpclib.Transport):
188 - def __init__(self, *args, **kwargs):
189 self.timeout = kwargs.pop("timeout", None) 190 xmlrpclib.Transport.__init__(self, *args, **kwargs)
191
192 - def make_connection(self, *args, **kwargs):
193 conn = xmlrpclib.Transport.make_connection(self, *args, **kwargs) 194 if self.timeout is not None: 195 conn.timeout = self.timeout 196 return conn
197
198 -class Singleton(type):
199 """Singleton. 200 @see: http://stackoverflow.com/questions/6760685/creating-a-singleton-in-python 201 """ 202 _instances = {} 203
204 - def __call__(cls, *args, **kwargs):
205 if cls not in cls._instances: 206 cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) 207 return cls._instances[cls]
208
209 -class ThreadSingleton(type):
210 """Singleton per thread.""" 211 _instances = threading.local() 212
213 - def __call__(cls, *args, **kwargs):
214 if not getattr(cls._instances, "instance", None): 215 cls._instances.instance = super(ThreadSingleton, cls).__call__(*args, **kwargs) 216 return cls._instances.instance
217
218 -def to_unicode(s):
219 """Attempt to fix non uft-8 string into utf-8. It tries to guess input encoding, 220 if fail retry with a replace strategy (so undetectable chars will be escaped). 221 @see: fuller list of encodings at http://docs.python.org/library/codecs.html#standard-encodings 222 """ 223 224 def brute_enc(s2): 225 """Trying to decode via simple brute forcing.""" 226 encodings = ("ascii", "utf8", "latin1") 227 for enc in encodings: 228 try: 229 return unicode(s2, enc) 230 except UnicodeDecodeError: 231 pass 232 return None
233 234 def chardet_enc(s2): 235 """Guess encoding via chardet.""" 236 enc = chardet.detect(s2)["encoding"] 237 238 try: 239 return unicode(s2, enc) 240 except UnicodeDecodeError: 241 pass 242 return None 243 244 # If already in unicode, skip. 245 if isinstance(s, unicode): 246 return s 247 248 # First try to decode against a little set of common encodings. 249 result = brute_enc(s) 250 251 # Try via chardet. 252 if not result and HAVE_CHARDET: 253 result = chardet_enc(s) 254 255 # If not possible to convert the input string, try again with 256 # a replace strategy. 257 if not result: 258 result = unicode(s, errors="replace") 259 260 return result 261
262 -def cleanup_value(v):
263 """Cleanup utility function, strips some unwanted parts from values.""" 264 v = str(v) 265 if v.startswith("\\??\\"): 266 v = v[4:] 267 return v
268
269 -def classlock(f):
270 """Classlock decorator (created for database.Database). 271 Used to put a lock to avoid sqlite errors. 272 """ 273 def inner(self, *args, **kwargs): 274 curframe = inspect.currentframe() 275 calframe = inspect.getouterframes(curframe, 2) 276 277 if calframe[1][1].endswith("database.py"): 278 return f(self, *args, **kwargs) 279 280 with self._lock: 281 return f(self, *args, **kwargs)
282 283 return inner 284
285 -class SuperLock(object):
286 - def __init__(self):
287 self.tlock = threading.Lock() 288 self.mlock = multiprocessing.Lock()
289
290 - def __enter__(self):
291 self.tlock.acquire() 292 self.mlock.acquire()
293
294 - def __exit__(self, type, value, traceback):
295 self.mlock.release() 296 self.tlock.release()
297
298 -def hash_file(method, filepath):
299 """Calculates an hash on a file by path. 300 @param method: callable hashing method 301 @param path: file path 302 @return: computed hash string 303 """ 304 f = open(filepath, "rb") 305 h = method() 306 while True: 307 buf = f.read(1024 * 1024) 308 if not buf: 309 break 310 h.update(buf) 311 return h.hexdigest()
312
313 -def md5_file(filepath):
314 return hash_file(hashlib.md5, filepath)
315
316 -def sha1_file(filepath):
317 return hash_file(hashlib.sha1, filepath)
318
319 -def sha256_file(filepath):
320 return hash_file(hashlib.sha256, filepath)
321 322 GUIDS = {} 323
324 -def guid_name(guid):
325 if not GUIDS: 326 for line in open(os.path.join(CUCKOO_ROOT, "data", "guids.txt")): 327 try: 328 guid, name, url = line.strip().split() 329 except: 330 log.debug("Invalid GUID entry: %s", line) 331 continue 332 333 GUIDS["{%s}" % guid] = name 334 335 return GUIDS.get(guid)
336
337 -def exception_message():
338 """Creates a message describing an unhandled exception.""" 339 def get_os_release(): 340 """Returns detailed OS release.""" 341 if platform.linux_distribution()[0]: 342 return " ".join(platform.linux_distribution()) 343 elif platform.mac_ver()[0]: 344 return "%s %s" % (platform.mac_ver()[0], platform.mac_ver()[2]) 345 else: 346 return "Unknown"
347 348 msg = ( 349 "Oops! Cuckoo failed in an unhandled exception!\nSometimes bugs are " 350 "already fixed in the development release, it is therefore " 351 "recommended to retry with the latest development release available " 352 "%s\nIf the error persists please open a new issue at %s\n\n" % 353 (GITHUB_URL, ISSUES_PAGE_URL) 354 ) 355 356 msg += "=== Exception details ===\n" 357 msg += "Cuckoo version: %s\n" % CUCKOO_VERSION 358 msg += "OS version: %s\n" % os.name 359 msg += "OS release: %s\n" % get_os_release() 360 msg += "Python version: %s\n" % sys.version.split()[0] 361 msg += "Machine arch: %s\n" % platform.machine() 362 363 git_version = os.path.join(CUCKOO_ROOT, ".git", "refs", "heads", "master") 364 if os.path.exists(git_version): 365 try: 366 msg += "Git version: %s\n" % open(git_version, "rb").read().strip() 367 except: 368 pass 369 370 try: 371 import pip 372 373 msg += "Modules: %s\n" % " ".join(sorted( 374 "%s:%s" % (package.key, package.version) 375 for package in pip.get_installed_distributions() 376 )) 377 except ImportError: 378 pass 379 380 msg += "\n" 381 return msg 382 383 _jsbeautify_blacklist = [ 384 "", 385 "error: Unknown p.a.c.k.e.r. encoding.\n", 386 ] 387 388 _jsbeautify_lock = threading.Lock() 389
390 -def jsbeautify(javascript):
391 """Beautifies Javascript through jsbeautifier and ignore some messages.""" 392 if not HAVE_JSBEAUTIFIER: 393 return javascript 394 395 with _jsbeautify_lock: 396 origout, sys.stdout = sys.stdout, StringIO() 397 javascript = jsbeautifier.beautify(javascript) 398 399 if sys.stdout.getvalue() not in _jsbeautify_blacklist: 400 log.warning("jsbeautifier returned error: %s", sys.stdout.getvalue()) 401 402 sys.stdout = origout 403 return javascript
404
405 -def htmlprettify(html):
406 """Beautifies HTML through BeautifulSoup4.""" 407 if not HAVE_BS4: 408 return html 409 410 # The following ignores the following bs4 warning: 411 # UserWarning: "." looks like a filename, not markup. 412 with warnings.catch_warnings(): 413 warnings.simplefilter("ignore", lineno=182) 414 return bs4.BeautifulSoup(html, "html.parser").prettify()
415
416 -def json_default(obj):
417 """JSON serializer for objects not serializable by default json code""" 418 if isinstance(obj, datetime): 419 if obj.utcoffset() is not None: 420 obj = obj - obj.utcoffset() 421 return {"$dt": obj.isoformat()} 422 raise TypeError("Type not serializable")
423
424 -def json_hook(obj):
425 """JSON object hook, deserializing datetimes ($date)""" 426 if "$dt" in obj: 427 x = obj["$dt"] 428 return datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%f") 429 return obj
430
431 -def json_encode(obj, **kwargs):
432 """JSON encoding wrapper that handles datetime objects""" 433 return json.dumps(obj, default=json_default, **kwargs)
434
435 -def json_decode(x):
436 """JSON decoder that does ugly first-level datetime handling""" 437 return json.loads(x, object_hook=json_hook)
438
439 -def versiontuple(v):
440 """Return the version as a tuple for easy comparison.""" 441 return tuple(int(x) for x in v.split("."))
442