Package modules :: Package processing :: Module network
[hide private]
[frames] | no frames]

Source Code for Module modules.processing.network

  1  # Copyright (C) 2010-2014 Cuckoo Foundation. 
  2  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 
  3  # See the file 'docs/LICENSE' for copying permission. 
  4   
  5  import os 
  6  import re 
  7  import struct 
  8  import socket 
  9  import logging 
 10  from urlparse import urlunparse 
 11   
 12  from lib.cuckoo.common.abstracts import Processing 
 13  from lib.cuckoo.common.config import Config 
 14  from lib.cuckoo.common.dns import resolve 
 15  from lib.cuckoo.common.irc import ircMessage 
 16  from lib.cuckoo.common.objects import File 
 17  from lib.cuckoo.common.utils import convert_to_printable 
 18   
 19  try: 
 20      import dpkt 
 21      IS_DPKT = True 
 22  except ImportError: 
 23      IS_DPKT = False 
 24   
25 -class Pcap:
26 """Reads network data from PCAP file.""" 27
28 - def __init__(self, filepath):
29 """Creates a new instance. 30 @param filepath: path to PCAP file 31 """ 32 self.filepath = filepath 33 34 # List of all hosts. 35 self.hosts = [] 36 # List containing all non-private IP addresses. 37 self.unique_hosts = [] 38 # List of unique domains. 39 self.unique_domains = [] 40 # List containing all TCP packets. 41 self.tcp_connections = [] 42 # List containing all UDP packets. 43 self.udp_connections = [] 44 # List containing all ICMP requests. 45 self.icmp_requests = [] 46 # List containing all HTTP requests. 47 self.http_requests = [] 48 # List containing all DNS requests. 49 self.dns_requests = [] 50 # List containing all SMTP requests. 51 self.smtp_requests = [] 52 # Reconstruncted SMTP flow. 53 self.smtp_flow = {} 54 # List containing all IRC requests. 55 self.irc_requests = [] 56 # Dictionary containing all the results of this processing. 57 self.results = {}
58
59 - def _dns_gethostbyname(self, name):
60 """Get host by name wrapper. 61 @param name: hostname. 62 @return: IP address or blank 63 """ 64 if Config().processing.resolve_dns: 65 ip = resolve(name) 66 else: 67 ip = "" 68 return ip
69
70 - def _is_private_ip(self, ip):
71 """Check if the IP belongs to private network blocks. 72 @param ip: IP address to verify. 73 @return: boolean representing whether the IP belongs or not to 74 a private network block. 75 """ 76 networks = [ 77 "0.0.0.0/8", 78 "10.0.0.0/8", 79 "100.64.0.0/10", 80 "127.0.0.0/8", 81 "169.254.0.0/16", 82 "172.16.0.0/12", 83 "192.0.0.0/24", 84 "192.0.2.0/24", 85 "192.88.99.0/24", 86 "192.168.0.0/16", 87 "198.18.0.0/15", 88 "198.51.100.0/24", 89 "203.0.113.0/24", 90 "240.0.0.0/4", 91 "255.255.255.255/32", 92 "224.0.0.0/4" 93 ] 94 95 for network in networks: 96 try: 97 ipaddr = struct.unpack(">I", socket.inet_aton(ip))[0] 98 99 netaddr, bits = network.split("/") 100 101 network_low = struct.unpack(">I", socket.inet_aton(netaddr))[0] 102 network_high = network_low | 1 << (32 - int(bits)) - 1 103 104 if ipaddr <= network_high and ipaddr >= network_low: 105 return True 106 except: 107 continue 108 109 return False
110
111 - def _add_hosts(self, connection):
112 """Add IPs to unique list. 113 @param connection: connection data 114 """ 115 try: 116 if connection["src"] not in self.hosts: 117 ip = convert_to_printable(connection["src"]) 118 if ip in self.hosts: 119 return 120 else: 121 self.hosts.append(ip) 122 123 if not self._is_private_ip(ip): 124 self.unique_hosts.append(ip) 125 126 if connection["dst"] not in self.hosts: 127 ip = convert_to_printable(connection["dst"]) 128 if ip in self.hosts: 129 return 130 else: 131 self.hosts.append(ip) 132 133 if not self._is_private_ip(ip): 134 self.unique_hosts.append(ip) 135 except: 136 pass
137
138 - def _tcp_dissect(self, conn, data):
139 """Runs all TCP dissectors. 140 @param conn: connection. 141 @param data: payload data. 142 """ 143 if self._check_http(data): 144 self._add_http(data, conn["dport"]) 145 # SMTP. 146 if conn["dport"] == 25: 147 self._reassemble_smtp(conn, data) 148 # IRC. 149 if conn["dport"] != 21 and self._check_irc(data): 150 self._add_irc(data)
151
152 - def _udp_dissect(self, conn, data):
153 """Runs all UDP dissectors. 154 @param conn: connection. 155 @param data: payload data. 156 """ 157 if conn["dport"] == 53 or conn["sport"] == 53: 158 if self._check_dns(data): 159 self._add_dns(data)
160
161 - def _check_icmp(self, icmp_data):
162 """Checks for ICMP traffic. 163 @param icmp_data: ICMP data flow. 164 """ 165 try: 166 return isinstance(icmp_data, dpkt.icmp.ICMP) and \ 167 len(icmp_data.data) > 0 168 except: 169 return False
170
171 - def _icmp_dissect(self, conn, data):
172 """Runs all ICMP dissectors. 173 @param conn: connection. 174 @param data: payload data. 175 """ 176 177 if self._check_icmp(data): 178 # If ICMP packets are coming from the host, it probably isn't 179 # relevant traffic, hence we can skip from reporting it. 180 if conn["src"] == Config().resultserver.ip: 181 return 182 183 entry = {} 184 entry["src"] = conn["src"] 185 entry["dst"] = conn["dst"] 186 entry["type"] = data.type 187 188 # Extract data from dpkg.icmp.ICMP. 189 try: 190 entry["data"] = convert_to_printable(data.data.data) 191 except: 192 entry["data"] = "" 193 194 self.icmp_requests.append(entry)
195
196 - def _check_dns(self, udpdata):
197 """Checks for DNS traffic. 198 @param udpdata: UDP data flow. 199 """ 200 try: 201 dpkt.dns.DNS(udpdata) 202 except: 203 return False 204 205 return True
206
207 - def _add_dns(self, udpdata):
208 """Adds a DNS data flow. 209 @param udpdata: UDP data flow. 210 """ 211 dns = dpkt.dns.DNS(udpdata) 212 213 # DNS query parsing. 214 query = {} 215 216 if dns.rcode == dpkt.dns.DNS_RCODE_NOERR or \ 217 dns.qr == dpkt.dns.DNS_R or \ 218 dns.opcode == dpkt.dns.DNS_QUERY or True: 219 # DNS question. 220 try: 221 q_name = dns.qd[0].name 222 q_type = dns.qd[0].type 223 except IndexError: 224 return False 225 226 query["request"] = q_name 227 if q_type == dpkt.dns.DNS_A: 228 query["type"] = "A" 229 if q_type == dpkt.dns.DNS_AAAA: 230 query["type"] = "AAAA" 231 elif q_type == dpkt.dns.DNS_CNAME: 232 query["type"] = "CNAME" 233 elif q_type == dpkt.dns.DNS_MX: 234 query["type"] = "MX" 235 elif q_type == dpkt.dns.DNS_PTR: 236 query["type"] = "PTR" 237 elif q_type == dpkt.dns.DNS_NS: 238 query["type"] = "NS" 239 elif q_type == dpkt.dns.DNS_SOA: 240 query["type"] = "SOA" 241 elif q_type == dpkt.dns.DNS_HINFO: 242 query["type"] = "HINFO" 243 elif q_type == dpkt.dns.DNS_TXT: 244 query["type"] = "TXT" 245 elif q_type == dpkt.dns.DNS_SRV: 246 query["type"] = "SRV" 247 248 # DNS answer. 249 query["answers"] = [] 250 for answer in dns.an: 251 ans = {} 252 if answer.type == dpkt.dns.DNS_A: 253 ans["type"] = "A" 254 try: 255 ans["data"] = socket.inet_ntoa(answer.rdata) 256 except socket.error: 257 continue 258 elif answer.type == dpkt.dns.DNS_AAAA: 259 ans["type"] = "AAAA" 260 try: 261 ans["data"] = socket.inet_ntop(socket.AF_INET6, 262 answer.rdata) 263 except (socket.error, ValueError): 264 continue 265 elif answer.type == dpkt.dns.DNS_CNAME: 266 ans["type"] = "CNAME" 267 ans["data"] = answer.cname 268 elif answer.type == dpkt.dns.DNS_MX: 269 ans["type"] = "MX" 270 ans["data"] = answer.mxname 271 elif answer.type == dpkt.dns.DNS_PTR: 272 ans["type"] = "PTR" 273 ans["data"] = answer.ptrname 274 elif answer.type == dpkt.dns.DNS_NS: 275 ans["type"] = "NS" 276 ans["data"] = answer.nsname 277 elif answer.type == dpkt.dns.DNS_SOA: 278 ans["type"] = "SOA" 279 ans["data"] = ",".join([answer.mname, 280 answer.rname, 281 str(answer.serial), 282 str(answer.refresh), 283 str(answer.retry), 284 str(answer.expire), 285 str(answer.minimum)]) 286 elif answer.type == dpkt.dns.DNS_HINFO: 287 ans["type"] = "HINFO" 288 ans["data"] = " ".join(answer.text) 289 elif answer.type == dpkt.dns.DNS_TXT: 290 ans["type"] = "TXT" 291 ans["data"] = " ".join(answer.text) 292 293 # TODO: add srv handling 294 query["answers"].append(ans) 295 296 self._add_domain(query["request"]) 297 self.dns_requests.append(query) 298 299 return True
300
301 - def _add_domain(self, domain):
302 """Add a domain to unique list. 303 @param domain: domain name. 304 """ 305 filters = [ 306 ".*\\.windows\\.com$", 307 ".*\\.in\\-addr\\.arpa$" 308 ] 309 310 regexps = [re.compile(filter) for filter in filters] 311 for regexp in regexps: 312 if regexp.match(domain): 313 return 314 315 for entry in self.unique_domains: 316 if entry["domain"] == domain: 317 return 318 319 self.unique_domains.append({"domain": domain, 320 "ip": self._dns_gethostbyname(domain)})
321
322 - def _check_http(self, tcpdata):
323 """Checks for HTTP traffic. 324 @param tcpdata: TCP data flow. 325 """ 326 try: 327 r = dpkt.http.Request() 328 r.method, r.version, r.uri = None, None, None 329 r.unpack(tcpdata) 330 except dpkt.dpkt.UnpackError: 331 if not r.method is None or not r.version is None or \ 332 not r.uri is None: 333 return True 334 return False 335 336 return True
337
338 - def _add_http(self, tcpdata, dport):
339 """Adds an HTTP flow. 340 @param tcpdata: TCP data flow. 341 @param dport: destination port. 342 """ 343 try: 344 http = dpkt.http.Request() 345 http.unpack(tcpdata) 346 except dpkt.dpkt.UnpackError: 347 pass 348 349 try: 350 entry = {} 351 352 if "host" in http.headers: 353 entry["host"] = convert_to_printable(http.headers["host"]) 354 else: 355 entry["host"] = "" 356 357 entry["port"] = dport 358 entry["data"] = convert_to_printable(tcpdata) 359 entry["uri"] = convert_to_printable(urlunparse(("http", 360 entry["host"], 361 http.uri, None, 362 None, None))) 363 entry["body"] = convert_to_printable(http.body) 364 entry["path"] = convert_to_printable(http.uri) 365 366 if "user-agent" in http.headers: 367 entry["user-agent"] = \ 368 convert_to_printable(http.headers["user-agent"]) 369 370 entry["version"] = convert_to_printable(http.version) 371 entry["method"] = convert_to_printable(http.method) 372 373 self.http_requests.append(entry) 374 except Exception: 375 return False 376 377 return True
378
379 - def _reassemble_smtp(self, conn, data):
380 """Reassemble a SMTP flow. 381 @param conn: connection dict. 382 @param data: raw data. 383 """ 384 if conn["dst"] in self.smtp_flow: 385 self.smtp_flow[conn["dst"]] += data 386 else: 387 self.smtp_flow[conn["dst"]] = data
388
389 - def _process_smtp(self):
390 """Process SMTP flow.""" 391 for conn, data in self.smtp_flow.iteritems(): 392 # Detect new SMTP flow. 393 if data.startswith("EHLO") or data.startswith("HELO"): 394 self.smtp_requests.append({"dst": conn, "raw": data})
395
396 - def _check_irc(self, tcpdata):
397 """ 398 Checks for IRC traffic. 399 @param tcpdata: tcp data flow 400 """ 401 try: 402 req = ircMessage() 403 except Exception: 404 return False 405 406 return req.isthereIRC(tcpdata)
407
408 - def _add_irc(self, tcpdata):
409 """ 410 Adds an IRC communication. 411 @param tcpdata: TCP data in flow 412 @param dport: destination port 413 """ 414 415 try: 416 reqc = ircMessage() 417 reqs = ircMessage() 418 filters_sc = ["266"] 419 self.irc_requests = self.irc_requests + \ 420 reqc.getClientMessages(tcpdata) + \ 421 reqs.getServerMessagesFilter(tcpdata, filters_sc) 422 except Exception: 423 return False 424 425 return True
426
427 - def run(self):
428 """Process PCAP. 429 @return: dict with network analysis data. 430 """ 431 log = logging.getLogger("Processing.Pcap") 432 433 if not IS_DPKT: 434 log.error("Python DPKT is not installed, aborting PCAP analysis.") 435 return self.results 436 437 if not os.path.exists(self.filepath): 438 log.warning("The PCAP file does not exist at path \"%s\".", 439 self.filepath) 440 return self.results 441 442 if os.path.getsize(self.filepath) == 0: 443 log.error("The PCAP file at path \"%s\" is empty." % self.filepath) 444 return self.results 445 446 try: 447 file = open(self.filepath, "rb") 448 except (IOError, OSError): 449 log.error("Unable to open %s" % self.filepath) 450 return self.results 451 452 try: 453 pcap = dpkt.pcap.Reader(file) 454 except dpkt.dpkt.NeedData: 455 log.error("Unable to read PCAP file at path \"%s\".", 456 self.filepath) 457 return self.results 458 except ValueError: 459 log.error("Unable to read PCAP file at path \"%s\". File is " 460 "corrupted or wrong format." % self.filepath) 461 return self.results 462 463 for ts, buf in pcap: 464 try: 465 eth = dpkt.ethernet.Ethernet(buf) 466 ip = eth.data 467 468 connection = {} 469 if isinstance(ip, dpkt.ip.IP): 470 connection["src"] = socket.inet_ntoa(ip.src) 471 connection["dst"] = socket.inet_ntoa(ip.dst) 472 elif isinstance(ip, dpkt.ip6.IP6): 473 connection["src"] = socket.inet_ntop(socket.AF_INET6, 474 ip.src) 475 connection["dst"] = socket.inet_ntop(socket.AF_INET6, 476 ip.dst) 477 else: 478 continue 479 480 self._add_hosts(connection) 481 482 if ip.p == dpkt.ip.IP_PROTO_TCP: 483 484 tcp = ip.data 485 486 if len(tcp.data) > 0: 487 connection["sport"] = tcp.sport 488 connection["dport"] = tcp.dport 489 self._tcp_dissect(connection, tcp.data) 490 self.tcp_connections.append(connection) 491 else: 492 continue 493 elif ip.p == dpkt.ip.IP_PROTO_UDP: 494 udp = ip.data 495 496 if len(udp.data) > 0: 497 connection["sport"] = udp.sport 498 connection["dport"] = udp.dport 499 self._udp_dissect(connection, udp.data) 500 self.udp_connections.append(connection) 501 elif ip.p == dpkt.ip.IP_PROTO_ICMP: 502 icmp = ip.data 503 self._icmp_dissect(connection, icmp) 504 except AttributeError: 505 continue 506 except dpkt.dpkt.NeedData: 507 continue 508 except Exception as e: 509 log.exception("Failed to process packet: %s", e) 510 511 file.close() 512 513 # Post processors for reconstructed flows. 514 self._process_smtp() 515 516 # Build results dict. 517 self.results["hosts"] = self.unique_hosts 518 self.results["domains"] = self.unique_domains 519 self.results["tcp"] = self.tcp_connections 520 self.results["udp"] = self.udp_connections 521 self.results["icmp"] = self.icmp_requests 522 self.results["http"] = self.http_requests 523 self.results["dns"] = self.dns_requests 524 self.results["smtp"] = self.smtp_requests 525 self.results["irc"] = self.irc_requests 526 527 return self.results
528
529 -class NetworkAnalysis(Processing):
530 """Network analysis.""" 531
532 - def run(self):
533 self.key = "network" 534 535 results = Pcap(self.pcap_path).run() 536 537 # Save PCAP file hash. 538 if os.path.exists(self.pcap_path): 539 results["pcap_sha256"] = File(self.pcap_path).get_sha256() 540 541 return results
542