1
2
3
4
5 import os
6 import logging
7 import datetime
8
9 from lib.cuckoo.common.abstracts import Processing
10 from lib.cuckoo.common.config import Config
11 from lib.cuckoo.common.netlog import NetlogParser, BsonParser
12 from lib.cuckoo.common.utils import convert_to_printable, logtime
13 from lib.cuckoo.common.utils import cleanup_value
14
15 log = logging.getLogger(__name__)
16
18 """Fix a registry key to have it normalized.
19 @param key: raw key
20 @returns: normalized key
21 """
22 res = key
23 if key.lower().startswith("registry\\machine\\"):
24 res = "HKEY_LOCAL_MACHINE\\" + key[17:]
25 elif key.lower().startswith("registry\\user\\"):
26 res = "HKEY_USERS\\" + key[14:]
27 elif key.lower().startswith("\\registry\\machine\\"):
28 res = "HKEY_LOCAL_MACHINE\\" + key[18:]
29 elif key.lower().startswith("\\registry\\user\\"):
30 res = "HKEY_USERS\\" + key[15:]
31
32 return res
33
35 """Parses process log file."""
36
38 """@param log_path: log file path."""
39 self._log_path = log_path
40 self.fd = None
41 self.parser = None
42
43 self.process_id = None
44 self.process_name = None
45 self.parent_id = None
46 self.first_seen = None
47 self.calls = self
48 self.lastcall = None
49
50 if os.path.exists(log_path) and os.stat(log_path).st_size > 0:
51 self.parse_first_and_reset()
52
54 self.fd = open(self._log_path, "rb")
55
56 if self._log_path.endswith(".bson"):
57 self.parser = BsonParser(self)
58 elif self._log_path.endswith(".raw"):
59 self.parser = NetlogParser(self)
60 else:
61 self.fd.close()
62 self.fd = None
63 return
64
65
66
67 while not self.process_id:
68 self.parser.read_next_message()
69
70 self.fd.seek(0)
71
72 - def read(self, length):
73 if not length:
74 return ''
75 buf = self.fd.read(length)
76 if not buf or len(buf) != length:
77 raise EOFError()
78 return buf
79
84
86 return "<ParseProcessLog log-path: %r>" % self._log_path
87
90
92 self.fd.seek(0)
93 self.lastcall = None
94
96 """Compare two calls for equality. Same implementation as before netlog.
97 @param a: call a
98 @param b: call b
99 @return: True if a == b else False
100 """
101 if a["api"] == b["api"] and \
102 a["status"] == b["status"] and \
103 a["arguments"] == b["arguments"] and \
104 a["return"] == b["return"]:
105 return True
106 return False
107
109 while not self.lastcall:
110 r = None
111 try:
112 r = self.parser.read_next_message()
113 except EOFError:
114 return False
115
116 if not r:
117 return False
118 return True
119
121 if not self.fd:
122 raise StopIteration()
123
124 if not self.wait_for_lastcall():
125 self.reset()
126 raise StopIteration()
127
128 nextcall, self.lastcall = self.lastcall, None
129
130 self.wait_for_lastcall()
131 while self.lastcall and self.compare_calls(nextcall, self.lastcall):
132 nextcall["repeated"] += 1
133 self.lastcall = None
134 self.wait_for_lastcall()
135
136 return nextcall
137
138 - def log_process(self, context, timestring, pid, ppid, modulepath, procname):
139 self.process_id, self.parent_id, self.process_name = pid, ppid, procname
140 self.first_seen = timestring
141
144
145 - def log_call(self, context, apiname, category, arguments):
146 apiindex, status, returnval, tid, timediff = context
147
148 current_time = self.first_seen + datetime.timedelta(0, 0, timediff*1000)
149 timestring = logtime(current_time)
150
151 self.lastcall = self._parse([timestring,
152 tid,
153 category,
154 apiname,
155 status,
156 returnval] + arguments)
157
159 log.warning("ParseProcessLog error condition on log %s: %s", str(self._log_path), emsg)
160
162 """Parse log row.
163 @param row: row data.
164 @return: parsed information dict.
165 """
166 call = {}
167 arguments = []
168
169 try:
170 timestamp = row[0]
171 thread_id = row[1]
172 category = row[2]
173 api_name = row[3]
174 status_value = row[4]
175 return_value = row[5]
176 except IndexError as e:
177 log.debug("Unable to parse process log row: %s", e)
178 return None
179
180
181
182 for index in range(6, len(row)):
183 argument = {}
184
185
186 try:
187 arg_name, arg_value = row[index]
188 except ValueError as e:
189 log.debug("Unable to parse analysis row argument (row=%s): %s", row[index], e)
190 continue
191
192 argument["name"] = arg_name
193
194 argument["value"] = convert_to_printable(cleanup_value(arg_value))
195 arguments.append(argument)
196
197 call["timestamp"] = timestamp
198 call["thread_id"] = str(thread_id)
199 call["category"] = category
200 call["api"] = api_name
201 call["status"] = bool(int(status_value))
202
203 if isinstance(return_value, int):
204 call["return"] = "0x%.08x" % return_value
205 else:
206 call["return"] = convert_to_printable(cleanup_value(return_value))
207
208 call["arguments"] = arguments
209 call["repeated"] = 0
210
211 return call
212
214 """Processes analyzer."""
215
217 """@param logs_path: logs path."""
218 self._logs_path = logs_path
219 self.cfg = Config()
220
222 """Run analysis.
223 @return: processes infomartion list.
224 """
225 results = []
226
227 if not os.path.exists(self._logs_path):
228 log.warning("Analysis results folder does not exist at path \"%s\".", self._logs_path)
229 return results
230
231 if len(os.listdir(self._logs_path)) == 0:
232 log.warning("Analysis results folder does not contain any file.")
233 return results
234
235 for file_name in os.listdir(self._logs_path):
236 file_path = os.path.join(self._logs_path, file_name)
237
238 if os.path.isdir(file_path):
239 continue
240
241
242 if os.stat(file_path).st_size > self.cfg.processing.analysis_size_limit:
243 log.warning("Behavioral log {0} too big to be processed, skipped.".format(file_name))
244 continue
245
246
247 current_log = ParseProcessLog(file_path)
248 if current_log.process_id is None:
249 continue
250
251
252
253 results.append({
254 "process_id": current_log.process_id,
255 "process_name": current_log.process_name,
256 "parent_id": current_log.parent_id,
257 "first_seen": logtime(current_log.first_seen),
258 "calls": current_log.calls,
259 })
260
261
262
263 results.sort(key=lambda process: process["first_seen"])
264
265 return results
266
268 """Generates summary information."""
269
270 key = "summary"
271
273 self.keys = []
274 self.mutexes = []
275 self.files = []
276 self.handles = []
277
279 for known_handle in self.handles:
280 if handle != 0 and handle == known_handle["handle"]:
281 return None
282
283 name = ""
284
285 if registry == 0x80000000:
286 name = "HKEY_CLASSES_ROOT\\"
287 elif registry == 0x80000001:
288 name = "HKEY_CURRENT_USER\\"
289 elif registry == 0x80000002:
290 name = "HKEY_LOCAL_MACHINE\\"
291 elif registry == 0x80000003:
292 name = "HKEY_USERS\\"
293 elif registry == 0x80000004:
294 name = "HKEY_PERFORMANCE_DATA\\"
295 elif registry == 0x80000005:
296 name = "HKEY_CURRENT_CONFIG\\"
297 elif registry == 0x80000006:
298 name = "HKEY_DYN_DATA\\"
299 else:
300 for known_handle in self.handles:
301 if registry == known_handle["handle"]:
302 name = known_handle["name"] + "\\"
303
304 key = fix_key(name + subkey)
305 self.handles.append({"handle": handle, "name": key})
306 return key
307
309 """Generate processes list from streamed calls/processes.
310 @return: None.
311 """
312
313 if call["api"].startswith("RegOpenKeyEx") or call["api"].startswith("RegCreateKeyEx"):
314 registry = 0
315 subkey = ""
316 handle = 0
317
318 for argument in call["arguments"]:
319 if argument["name"] == "Registry":
320 registry = int(argument["value"], 16)
321 elif argument["name"] == "SubKey":
322 subkey = argument["value"]
323 elif argument["name"] == "Handle":
324 handle = int(argument["value"], 16)
325
326 name = self._check_registry(registry, subkey, handle)
327 if name and name not in self.keys:
328 self.keys.append(name)
329 elif call["api"].startswith("NtOpenKey"):
330 registry = -1
331 subkey = ""
332 handle = 0
333
334 for argument in call["arguments"]:
335 if argument["name"] == "ObjectAttributes":
336 subkey = argument["value"]
337 elif argument["name"] == "KeyHandle":
338 handle = int(argument["value"], 16)
339
340 name = self._check_registry(registry, subkey, handle)
341 if name and name not in self.keys:
342 self.keys.append(name)
343 elif call["api"].startswith("NtDeleteValueKey"):
344 registry = -1
345 subkey = ""
346 handle = 0
347
348 for argument in call["arguments"]:
349 if argument["name"] == "ValueName":
350 subkey = argument["value"]
351 elif argument["name"] == "KeyHandle":
352 handle = int(argument["value"], 16)
353
354 name = self._check_registry(registry, subkey, handle)
355 if name and name not in self.keys:
356 self.keys.append(name)
357 elif call["api"].startswith("RegCloseKey"):
358 handle = 0
359
360 for argument in call["arguments"]:
361 if argument["name"] == "Handle":
362 handle = int(argument["value"], 16)
363
364 if handle != 0:
365 for a in self.handles:
366 if a["handle"] == handle:
367 try:
368 self.handles.remove(a)
369 except ValueError:
370 pass
371
372 elif call["category"] == "filesystem":
373 for argument in call["arguments"]:
374 if argument["name"] == "FileName":
375 value = argument["value"].strip()
376 if not value:
377 continue
378
379 if value not in self.files:
380 self.files.append(value)
381
382 elif call["category"] == "synchronization":
383 for argument in call["arguments"]:
384 if argument["name"] == "MutexName":
385 value = argument["value"].strip()
386 if not value:
387 continue
388
389 if value not in self.mutexes:
390 self.mutexes.append(value)
391
393 """Get registry keys, mutexes and files.
394 @return: Summary of keys, mutexes and files.
395 """
396 return {"files": self.files, "keys": self.keys, "mutexes": self.mutexes}
397
399 """Generates a more extensive high-level representation than Summary."""
400
401 key = "enhanced"
402
404 """
405 @param details: Also add some (not so relevant) Details to the log
406 """
407 self.currentdir = "C: "
408 self.eid = 0
409 self.details = details
410 self.filehandles = {}
411 self.servicehandles = {}
412 self.keyhandles = {
413 "0x80000000": "HKEY_CLASSES_ROOT\\",
414 "0x80000001": "HKEY_CURRENT_USER\\",
415 "0x80000002": "HKEY_LOCAL_MACHINE\\",
416 "0x80000003": "HKEY_USERS\\",
417 "0x80000004": "HKEY_PERFORMANCE_DATA\\",
418 "0x80000005": "HKEY_CURRENT_CONFIG\\",
419 "0x80000006": "HKEY_DYN_DATA\\"
420 }
421 self.modules = {}
422 self.procedures = {}
423 self.events = []
424
426 """
427 Add a procedure address
428 """
429 self.procedures[base] = "{0}:{1}".format(self._get_loaded_module(mbase), name)
430
432 """
433 Add a loaded module to the internal database
434 """
435 self.modules[base] = name
436
438 """
439 Get the name of a loaded module from the internal db
440 """
441 return self.modules.get(base, "")
442
443
445 """
446 @registry: returned, new handle
447 @handle: handle to base key
448 @subkey: subkey to add
449 """
450 if handle != 0 and handle in self.keyhandles:
451 return self.keyhandles[handle]
452
453 name = ""
454 if registry and registry != "0x00000000" and \
455 registry in self.keyhandles:
456 name = self.keyhandles[registry]
457
458 nkey = name + subkey
459 nkey = fix_key(nkey)
460
461 self.keyhandles[handle] = nkey
462
463 return nkey
464
472
475
477 """ Gets files calls
478 @return: information list
479 """
480 def _load_args(call):
481 """
482 Load arguments from call
483 """
484 res = {}
485 for argument in call["arguments"]:
486 res[argument["name"]] = argument["value"]
487
488 return res
489
490 def _generic_handle_details(self, call, item):
491 """
492 Generic handling of api calls
493 @call: the call dict
494 @item: Generic item to process
495 """
496 event = None
497 if call["api"] in item["apis"]:
498 args = _load_args(call)
499 self.eid += 1
500
501 event = {
502 "event": item["event"],
503 "object": item["object"],
504 "timestamp": call["timestamp"],
505 "eid": self.eid,
506 "data": {}
507 }
508
509 for logname, dataname in item["args"]:
510 event["data"][logname] = args.get(dataname)
511 return event
512
513 def _generic_handle(self, data, call):
514 """Generic handling of api calls."""
515 for item in data:
516 event = _generic_handle_details(self, call, item)
517 if event:
518 return event
519
520 return None
521
522
523 def _add_handle(handles, handle, filename):
524 handles[handle] = filename
525
526 def _remove_handle(handles, handle):
527 if handle in handles:
528 handles.pop(handle)
529
530 def _get_handle(handles, handle):
531 return handles.get(handle)
532
533 def _get_service_action(control_code):
534 """@see: http://msdn.microsoft.com/en-us/library/windows/desktop/ms682108%28v=vs.85%29.aspx"""
535 codes = {1: "stop",
536 2: "pause",
537 3: "continue",
538 4: "info"}
539
540 default = "user" if control_code >= 128 else "notify"
541 return codes.get(control_code, default)
542
543 event = None
544
545 gendat = [
546 {
547 "event": "move",
548 "object": "file",
549 "apis": [
550 "MoveFileWithProgressW",
551 "MoveFileExA",
552 "MoveFileExW"
553 ],
554 "args": [
555 ("from", "ExistingFileName"),
556 ("to", "NewFileName")
557 ]
558 },
559 {
560 "event": "copy",
561 "object": "file",
562 "apis": [
563 "CopyFileA",
564 "CopyFileW",
565 "CopyFileExW",
566 "CopyFileExA"
567 ],
568 "args": [
569 ("from", "ExistingFileName"),
570 ("to", "NewFileName")
571 ]
572 },
573 {
574 "event": "delete",
575 "object": "file",
576 "apis": [
577 "DeleteFileA",
578 "DeleteFileW",
579 "NtDeleteFile"
580 ],
581 "args": [("file", "FileName")]
582 },
583 {
584 "event": "delete",
585 "object": "dir",
586 "apis": [
587 "RemoveDirectoryA",
588 "RemoveDirectoryW"
589 ],
590 "args": [("file", "DirectoryName")]
591 },
592 {
593 "event": "create",
594 "object": "dir",
595 "apis": [
596 "CreateDirectoryW",
597 "CreateDirectoryExW"
598 ],
599 "args": [("file", "DirectoryName")]
600 },
601 {
602 "event": "write",
603 "object": "file",
604 "apis": [
605 "URLDownloadToFileW",
606 "URLDownloadToFileA"
607 ],
608 "args": [("file", "FileName")]
609 },
610 {
611 "event": "execute",
612 "object": "file",
613 "apis": [
614 "CreateProcessAsUserA",
615 "CreateProcessAsUserW",
616 "CreateProcessA",
617 "CreateProcessW",
618 "NtCreateProcess",
619 "NtCreateProcessEx"
620 ],
621 "args": [("file", "FileName")]
622 },
623 {
624 "event": "execute",
625 "object": "file",
626 "apis": [
627 "CreateProcessInternalW",
628 ],
629 "args": [("file", "CommandLine")]
630 },
631 {
632 "event": "execute",
633 "object": "file",
634 "apis": [
635 "ShellExecuteExA",
636 "ShellExecuteExW",
637 ],
638 "args": [("file", "FilePath")]
639 },
640 {
641 "event": "load",
642 "object": "library",
643 "apis": [
644 "LoadLibraryA",
645 "LoadLibraryW",
646 "LoadLibraryExA",
647 "LoadLibraryExW",
648 "LdrLoadDll",
649 "LdrGetDllHandle"
650 ],
651 "args": [
652 ("file", "FileName"),
653 ("pathtofile", "PathToFile"),
654 ("moduleaddress", "BaseAddress")
655 ]
656 },
657 {
658 "event": "findwindow",
659 "object": "windowname",
660 "apis": [
661 "FindWindowA",
662 "FindWindowW",
663 "FindWindowExA",
664 "FindWindowExW"
665 ],
666 "args": [
667 ("classname", "ClassName"),
668 ("windowname", "WindowName")
669 ]
670 },
671 {
672 "event": "read",
673 "object": "file",
674 "apis": [
675 "NtReadFile",
676 "ReadFile"
677 ],
678 "args": []
679 },
680 {
681 "event": "write",
682 "object": "file",
683 "apis": ["NtWriteFile"],
684 "args": []
685 },
686 {
687 "event": "delete",
688 "object": "registry",
689 "apis": [
690 "RegDeleteKeyA",
691 "RegDeleteKeyW"
692 ],
693 "args": []
694 },
695 {
696 "event": "write",
697 "object": "registry",
698 "apis": [
699 "RegSetValueExA",
700 "RegSetValueExW"
701 ],
702 "args": [
703 ("content", "Buffer"),
704 ("object", "object")
705 ]
706 },
707 {
708 "event": "read",
709 "object": "registry",
710 "apis": [
711 "RegQueryValueExA",
712 "RegQueryValueExW",
713 "NtQueryValueKey"
714 ],
715 "args": []
716 },
717 {
718 "event": "delete",
719 "object": "registry",
720 "apis": [
721 "RegDeleteValueA",
722 "RegDeleteValueW",
723 "NtDeleteValueKey"
724 ],
725 "args": []
726 },
727 {
728 "event": "create",
729 "object": "windowshook",
730 "apis": ["SetWindowsHookExA"],
731 "args": [
732 ("id", "HookIdentifier"),
733 ("moduleaddress", "ModuleAddress"),
734 ("procedureaddress", "ProcedureAddress")
735 ]
736 },
737 {
738 "event": "modify",
739 "object": "service",
740 "apis": ["ControlService"],
741 "args": [("controlcode", "ControlCode")]
742 },
743 {
744 "event": "delete",
745 "object": "service",
746 "apis": ["DeleteService"],
747 "args": [],
748 },
749 ]
750
751
752
753
754
755
756
757
758
759
760 event = _generic_handle(self, gendat, call)
761 args = _load_args(call)
762
763 if event:
764 if call["api"] in ["NtReadFile", "ReadFile", "NtWriteFile"]:
765 event["data"]["file"] = _get_handle(self.filehandles, args["FileHandle"])
766
767 elif call["api"] in ["RegDeleteKeyA", "RegDeleteKeyW"]:
768 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("SubKey", ""))
769
770 elif call["api"] in ["RegSetValueExA", "RegSetValueExW"]:
771 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("ValueName", ""))
772
773 elif call["api"] in ["RegQueryValueExA", "RegQueryValueExW", "RegDeleteValueA", "RegDeleteValueW"]:
774 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "UNKNOWN")), args.get("ValueName", ""))
775
776 elif call["api"] in ["NtQueryValueKey", "NtDeleteValueKey"]:
777 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("KeyHandle", "UNKNOWN")), args.get("ValueName", ""))
778
779 elif call["api"] in ["LoadLibraryA", "LoadLibraryW", "LoadLibraryExA", "LoadLibraryExW", "LdrGetDllHandle"] and call["status"]:
780 self._add_loaded_module(args.get("FileName", ""), args.get("ModuleHandle", ""))
781
782 elif call["api"] in ["LdrLoadDll"] and call["status"]:
783 self._add_loaded_module(args.get("FileName", ""), args.get("BaseAddress", ""))
784
785 elif call["api"] in ["LdrGetProcedureAddress"] and call["status"]:
786 self._add_procedure(args.get("ModuleHandle", ""), args.get("FunctionName", ""), args.get("FunctionAddress", ""))
787 event["data"]["module"] = self._get_loaded_module(args.get("ModuleHandle", ""))
788
789 elif call["api"] in ["SetWindowsHookExA"]:
790 event["data"]["module"] = self._get_loaded_module(args.get("ModuleAddress", ""))
791
792 if call["api"] in ["ControlService", "DeleteService"]:
793 event["data"]["service"] = _get_handle(self.servicehandles, args["ServiceHandle"])
794
795 if call["api"] in ["ControlService"]:
796 event["data"]["action"] = _get_service_action(args["ControlCode"])
797
798 return event
799
800 elif call["api"] in ["SetCurrentDirectoryA", "SetCurrentDirectoryW"]:
801 self.currentdir = args["Path"]
802
803
804 elif call["api"] in ["NtCreateFile", "NtOpenFile"]:
805 _add_handle(self.filehandles, args["FileHandle"], args["FileName"])
806
807 elif call["api"] in ["CreateFileW"]:
808 _add_handle(self.filehandles, call["return"], args["FileName"])
809
810 elif call["api"] in ["NtClose", "CloseHandle"]:
811 _remove_handle(self.filehandles, args["Handle"])
812
813
814 elif call["api"] in ["OpenServiceW"]:
815 _add_handle(self.servicehandles, call["return"], args["ServiceName"])
816
817
818 elif call["api"] in ["RegOpenKeyExA", "RegOpenKeyExW", "RegCreateKeyExA", "RegCreateKeyExW"]:
819 self._add_keyhandle(args.get("Registry", ""), args.get("SubKey", ""), args.get("Handle", ""))
820
821 elif call["api"] in ["NtOpenKey"]:
822 self._add_keyhandle(None, args.get("ObjectAttributes", ""), args.get("KeyHandle", ""))
823
824 elif call["api"] in ["RegCloseKey"]:
825 self._remove_keyhandle(args.get("Handle", ""))
826
827 return event
828
830 """Generate processes list from streamed calls/processes.
831 @return: None.
832 """
833 event = self._process_call(call)
834 if event:
835 self.events.append(event)
836
838 """Get registry keys, mutexes and files.
839 @return: Summary of keys, mutexes and files.
840 """
841 return self.events
842
844 """Generates process tree."""
845
846 key = "processtree"
847
849 self.processes = []
850 self.tree = []
851
853 """Add a node to a process tree.
854 @param node: node to add.
855 @param tree: processes tree.
856 @return: boolean with operation success status.
857 """
858
859 for process in tree:
860
861
862 if process["pid"] == node["parent_id"]:
863 process["children"].append(node)
864
865 else:
866 self.add_node(node, process["children"])
867
869 for entry in self.processes:
870 if entry["pid"] == process["process_id"]:
871 return
872
873 self.processes.append(dict(
874 name=process["process_name"],
875 pid=process["process_id"],
876 parent_id=process["parent_id"],
877 children=[]
878 ))
879
881 children = []
882
883
884 for process in self.processes:
885 has_parent = False
886
887 for process_again in self.processes:
888
889
890 if process_again["pid"] == process["parent_id"]:
891 has_parent = True
892
893
894 if has_parent:
895 children.append(process)
896
897 else:
898 self.tree.append(process)
899
900
901 for process in children:
902 self.add_node(process, self.tree)
903
904 return self.tree
905
907 """Behavior Analyzer."""
908
909 key = "behavior"
910
944