1
2
3
4
5 import os
6 import re
7 import struct
8 import socket
9 import logging
10 from urlparse import urlunparse
11
12 from lib.cuckoo.common.abstracts import Processing
13 from lib.cuckoo.common.config import Config
14 from lib.cuckoo.common.dns import resolve
15 from lib.cuckoo.common.irc import ircMessage
16 from lib.cuckoo.common.objects import File
17 from lib.cuckoo.common.utils import convert_to_printable
18
19 try:
20 import dpkt
21 IS_DPKT = True
22 except ImportError:
23 IS_DPKT = False
24
26 """Reads network data from PCAP file."""
27
29 """Creates a new instance.
30 @param filepath: path to PCAP file
31 """
32 self.filepath = filepath
33
34
35 self.hosts = []
36
37 self.unique_hosts = []
38
39 self.unique_domains = []
40
41 self.tcp_connections = []
42
43 self.udp_connections = []
44
45 self.icmp_requests = []
46
47 self.http_requests = []
48
49 self.dns_requests = []
50
51 self.smtp_requests = []
52
53 self.smtp_flow = {}
54
55 self.irc_requests = []
56
57 self.results = {}
58
60 """Get host by name wrapper.
61 @param name: hostname.
62 @return: IP address or blank
63 """
64 if Config().processing.resolve_dns:
65 ip = resolve(name)
66 else:
67 ip = ""
68 return ip
69
71 """Check if the IP belongs to private network blocks.
72 @param ip: IP address to verify.
73 @return: boolean representing whether the IP belongs or not to
74 a private network block.
75 """
76 networks = [
77 "0.0.0.0/8",
78 "10.0.0.0/8",
79 "100.64.0.0/10",
80 "127.0.0.0/8",
81 "169.254.0.0/16",
82 "172.16.0.0/12",
83 "192.0.0.0/24",
84 "192.0.2.0/24",
85 "192.88.99.0/24",
86 "192.168.0.0/16",
87 "198.18.0.0/15",
88 "198.51.100.0/24",
89 "203.0.113.0/24",
90 "240.0.0.0/4",
91 "255.255.255.255/32",
92 "224.0.0.0/4"
93 ]
94
95 for network in networks:
96 try:
97 ipaddr = struct.unpack(">I", socket.inet_aton(ip))[0]
98
99 netaddr, bits = network.split("/")
100
101 network_low = struct.unpack(">I", socket.inet_aton(netaddr))[0]
102 network_high = network_low | 1 << (32 - int(bits)) - 1
103
104 if ipaddr <= network_high and ipaddr >= network_low:
105 return True
106 except:
107 continue
108
109 return False
110
112 """Add IPs to unique list.
113 @param connection: connection data
114 """
115 try:
116 if connection["src"] not in self.hosts:
117 ip = convert_to_printable(connection["src"])
118 if ip in self.hosts:
119 return
120 else:
121 self.hosts.append(ip)
122
123 if not self._is_private_ip(ip):
124 self.unique_hosts.append(ip)
125
126 if connection["dst"] not in self.hosts:
127 ip = convert_to_printable(connection["dst"])
128 if ip in self.hosts:
129 return
130 else:
131 self.hosts.append(ip)
132
133 if not self._is_private_ip(ip):
134 self.unique_hosts.append(ip)
135 except:
136 pass
137
139 """Runs all TCP dissectors.
140 @param conn: connection.
141 @param data: payload data.
142 """
143 if self._check_http(data):
144 self._add_http(data, conn["dport"])
145
146 if conn["dport"] == 25:
147 self._reassemble_smtp(conn, data)
148
149 if conn["dport"] != 21 and self._check_irc(data):
150 self._add_irc(data)
151
153 """Runs all UDP dissectors.
154 @param conn: connection.
155 @param data: payload data.
156 """
157 if conn["dport"] == 53 or conn["sport"] == 53:
158 if self._check_dns(data):
159 self._add_dns(data)
160
162 """Checks for ICMP traffic.
163 @param icmp_data: ICMP data flow.
164 """
165 try:
166 return isinstance(icmp_data, dpkt.icmp.ICMP) and \
167 len(icmp_data.data) > 0
168 except:
169 return False
170
172 """Runs all ICMP dissectors.
173 @param conn: connection.
174 @param data: payload data.
175 """
176
177 if self._check_icmp(data):
178
179
180 if conn["src"] == Config().resultserver.ip:
181 return
182
183 entry = {}
184 entry["src"] = conn["src"]
185 entry["dst"] = conn["dst"]
186 entry["type"] = data.type
187
188
189 try:
190 entry["data"] = convert_to_printable(data.data.data)
191 except:
192 entry["data"] = ""
193
194 self.icmp_requests.append(entry)
195
197 """Checks for DNS traffic.
198 @param udpdata: UDP data flow.
199 """
200 try:
201 dpkt.dns.DNS(udpdata)
202 except:
203 return False
204
205 return True
206
208 """Adds a DNS data flow.
209 @param udpdata: UDP data flow.
210 """
211 dns = dpkt.dns.DNS(udpdata)
212
213
214 query = {}
215
216 if dns.rcode == dpkt.dns.DNS_RCODE_NOERR or \
217 dns.qr == dpkt.dns.DNS_R or \
218 dns.opcode == dpkt.dns.DNS_QUERY or True:
219
220 try:
221 q_name = dns.qd[0].name
222 q_type = dns.qd[0].type
223 except IndexError:
224 return False
225
226 query["request"] = q_name
227 if q_type == dpkt.dns.DNS_A:
228 query["type"] = "A"
229 if q_type == dpkt.dns.DNS_AAAA:
230 query["type"] = "AAAA"
231 elif q_type == dpkt.dns.DNS_CNAME:
232 query["type"] = "CNAME"
233 elif q_type == dpkt.dns.DNS_MX:
234 query["type"] = "MX"
235 elif q_type == dpkt.dns.DNS_PTR:
236 query["type"] = "PTR"
237 elif q_type == dpkt.dns.DNS_NS:
238 query["type"] = "NS"
239 elif q_type == dpkt.dns.DNS_SOA:
240 query["type"] = "SOA"
241 elif q_type == dpkt.dns.DNS_HINFO:
242 query["type"] = "HINFO"
243 elif q_type == dpkt.dns.DNS_TXT:
244 query["type"] = "TXT"
245 elif q_type == dpkt.dns.DNS_SRV:
246 query["type"] = "SRV"
247
248
249 query["answers"] = []
250 for answer in dns.an:
251 ans = {}
252 if answer.type == dpkt.dns.DNS_A:
253 ans["type"] = "A"
254 try:
255 ans["data"] = socket.inet_ntoa(answer.rdata)
256 except socket.error:
257 continue
258 elif answer.type == dpkt.dns.DNS_AAAA:
259 ans["type"] = "AAAA"
260 try:
261 ans["data"] = socket.inet_ntop(socket.AF_INET6,
262 answer.rdata)
263 except (socket.error, ValueError):
264 continue
265 elif answer.type == dpkt.dns.DNS_CNAME:
266 ans["type"] = "CNAME"
267 ans["data"] = answer.cname
268 elif answer.type == dpkt.dns.DNS_MX:
269 ans["type"] = "MX"
270 ans["data"] = answer.mxname
271 elif answer.type == dpkt.dns.DNS_PTR:
272 ans["type"] = "PTR"
273 ans["data"] = answer.ptrname
274 elif answer.type == dpkt.dns.DNS_NS:
275 ans["type"] = "NS"
276 ans["data"] = answer.nsname
277 elif answer.type == dpkt.dns.DNS_SOA:
278 ans["type"] = "SOA"
279 ans["data"] = ",".join([answer.mname,
280 answer.rname,
281 str(answer.serial),
282 str(answer.refresh),
283 str(answer.retry),
284 str(answer.expire),
285 str(answer.minimum)])
286 elif answer.type == dpkt.dns.DNS_HINFO:
287 ans["type"] = "HINFO"
288 ans["data"] = " ".join(answer.text)
289 elif answer.type == dpkt.dns.DNS_TXT:
290 ans["type"] = "TXT"
291 ans["data"] = " ".join(answer.text)
292
293
294 query["answers"].append(ans)
295
296 self._add_domain(query["request"])
297 self.dns_requests.append(query)
298
299 return True
300
301 - def _add_domain(self, domain):
302 """Add a domain to unique list.
303 @param domain: domain name.
304 """
305 filters = [
306 ".*\\.windows\\.com$",
307 ".*\\.in\\-addr\\.arpa$"
308 ]
309
310 regexps = [re.compile(filter) for filter in filters]
311 for regexp in regexps:
312 if regexp.match(domain):
313 return
314
315 for entry in self.unique_domains:
316 if entry["domain"] == domain:
317 return
318
319 self.unique_domains.append({"domain": domain,
320 "ip": self._dns_gethostbyname(domain)})
321
323 """Checks for HTTP traffic.
324 @param tcpdata: TCP data flow.
325 """
326 try:
327 r = dpkt.http.Request()
328 r.method, r.version, r.uri = None, None, None
329 r.unpack(tcpdata)
330 except dpkt.dpkt.UnpackError:
331 if not r.method is None or not r.version is None or \
332 not r.uri is None:
333 return True
334 return False
335
336 return True
337
339 """Adds an HTTP flow.
340 @param tcpdata: TCP data flow.
341 @param dport: destination port.
342 """
343 try:
344 http = dpkt.http.Request()
345 http.unpack(tcpdata)
346 except dpkt.dpkt.UnpackError:
347 pass
348
349 try:
350 entry = {}
351
352 if "host" in http.headers:
353 entry["host"] = convert_to_printable(http.headers["host"])
354 else:
355 entry["host"] = ""
356
357 entry["port"] = dport
358 entry["data"] = convert_to_printable(tcpdata)
359 entry["uri"] = convert_to_printable(urlunparse(("http",
360 entry["host"],
361 http.uri, None,
362 None, None)))
363 entry["body"] = convert_to_printable(http.body)
364 entry["path"] = convert_to_printable(http.uri)
365
366 if "user-agent" in http.headers:
367 entry["user-agent"] = \
368 convert_to_printable(http.headers["user-agent"])
369
370 entry["version"] = convert_to_printable(http.version)
371 entry["method"] = convert_to_printable(http.method)
372
373 self.http_requests.append(entry)
374 except Exception:
375 return False
376
377 return True
378
380 """Reassemble a SMTP flow.
381 @param conn: connection dict.
382 @param data: raw data.
383 """
384 if conn["dst"] in self.smtp_flow:
385 self.smtp_flow[conn["dst"]] += data
386 else:
387 self.smtp_flow[conn["dst"]] = data
388
390 """Process SMTP flow."""
391 for conn, data in self.smtp_flow.iteritems():
392
393 if data.startswith("EHLO") or data.startswith("HELO"):
394 self.smtp_requests.append({"dst": conn, "raw": data})
395
397 """
398 Checks for IRC traffic.
399 @param tcpdata: tcp data flow
400 """
401 try:
402 req = ircMessage()
403 except Exception:
404 return False
405
406 return req.isthereIRC(tcpdata)
407
409 """
410 Adds an IRC communication.
411 @param tcpdata: TCP data in flow
412 @param dport: destination port
413 """
414
415 try:
416 reqc = ircMessage()
417 reqs = ircMessage()
418 filters_sc = ["266"]
419 self.irc_requests = self.irc_requests + \
420 reqc.getClientMessages(tcpdata) + \
421 reqs.getServerMessagesFilter(tcpdata, filters_sc)
422 except Exception:
423 return False
424
425 return True
426
428 """Process PCAP.
429 @return: dict with network analysis data.
430 """
431 log = logging.getLogger("Processing.Pcap")
432
433 if not IS_DPKT:
434 log.error("Python DPKT is not installed, aborting PCAP analysis.")
435 return self.results
436
437 if not os.path.exists(self.filepath):
438 log.warning("The PCAP file does not exist at path \"%s\".",
439 self.filepath)
440 return self.results
441
442 if os.path.getsize(self.filepath) == 0:
443 log.error("The PCAP file at path \"%s\" is empty." % self.filepath)
444 return self.results
445
446 try:
447 file = open(self.filepath, "rb")
448 except (IOError, OSError):
449 log.error("Unable to open %s" % self.filepath)
450 return self.results
451
452 try:
453 pcap = dpkt.pcap.Reader(file)
454 except dpkt.dpkt.NeedData:
455 log.error("Unable to read PCAP file at path \"%s\".",
456 self.filepath)
457 return self.results
458 except ValueError:
459 log.error("Unable to read PCAP file at path \"%s\". File is "
460 "corrupted or wrong format." % self.filepath)
461 return self.results
462
463 for ts, buf in pcap:
464 try:
465 eth = dpkt.ethernet.Ethernet(buf)
466 ip = eth.data
467
468 connection = {}
469 if isinstance(ip, dpkt.ip.IP):
470 connection["src"] = socket.inet_ntoa(ip.src)
471 connection["dst"] = socket.inet_ntoa(ip.dst)
472 elif isinstance(ip, dpkt.ip6.IP6):
473 connection["src"] = socket.inet_ntop(socket.AF_INET6,
474 ip.src)
475 connection["dst"] = socket.inet_ntop(socket.AF_INET6,
476 ip.dst)
477 else:
478 continue
479
480 self._add_hosts(connection)
481
482 if ip.p == dpkt.ip.IP_PROTO_TCP:
483
484 tcp = ip.data
485
486 if len(tcp.data) > 0:
487 connection["sport"] = tcp.sport
488 connection["dport"] = tcp.dport
489 self._tcp_dissect(connection, tcp.data)
490 self.tcp_connections.append(connection)
491 else:
492 continue
493 elif ip.p == dpkt.ip.IP_PROTO_UDP:
494 udp = ip.data
495
496 if len(udp.data) > 0:
497 connection["sport"] = udp.sport
498 connection["dport"] = udp.dport
499 self._udp_dissect(connection, udp.data)
500 self.udp_connections.append(connection)
501 elif ip.p == dpkt.ip.IP_PROTO_ICMP:
502 icmp = ip.data
503 self._icmp_dissect(connection, icmp)
504 except AttributeError:
505 continue
506 except dpkt.dpkt.NeedData:
507 continue
508 except Exception as e:
509 log.exception("Failed to process packet: %s", e)
510
511 file.close()
512
513
514 self._process_smtp()
515
516
517 self.results["hosts"] = self.unique_hosts
518 self.results["domains"] = self.unique_domains
519 self.results["tcp"] = self.tcp_connections
520 self.results["udp"] = self.udp_connections
521 self.results["icmp"] = self.icmp_requests
522 self.results["http"] = self.http_requests
523 self.results["dns"] = self.dns_requests
524 self.results["smtp"] = self.smtp_requests
525 self.results["irc"] = self.irc_requests
526
527 return self.results
528
530 """Network analysis."""
531
533 self.key = "network"
534
535 results = Pcap(self.pcap_path).run()
536
537
538 if os.path.exists(self.pcap_path):
539 results["pcap_sha256"] = File(self.pcap_path).get_sha256()
540
541 return results
542