Statistics.py 66 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495
  1. import os
  2. import random
  3. import time
  4. from math import sqrt, ceil, log
  5. from operator import itemgetter
  6. # TODO: double check this import
  7. # does it complain because libpcapreader is not a .py?
  8. import ID2TLib.libpcapreader as pr
  9. import matplotlib
  10. import Core.StatsDatabase as statsDB
  11. import ID2TLib.PcapFile as PcapFile
  12. import ID2TLib.Utility as Util
  13. from ID2TLib.IPv4 import IPAddress
  14. matplotlib.use('Agg', force=True)
  15. import matplotlib.pyplot as plt
  16. class Statistics:
  17. def __init__(self, pcap_file: PcapFile.PcapFile):
  18. """
  19. Creates a new Statistics object.
  20. :param pcap_file: A reference to the PcapFile object
  21. """
  22. # Fields
  23. self.pcap_filepath = pcap_file.pcap_file_path
  24. self.pcap_proc = None
  25. self.do_extra_tests = False
  26. self.file_info = None
  27. # Create folder for statistics database if required
  28. self.path_db = pcap_file.get_db_path()
  29. path_dir = os.path.dirname(self.path_db)
  30. if not os.path.isdir(path_dir):
  31. os.makedirs(path_dir)
  32. # Class instances
  33. self.stats_db = statsDB.StatsDatabase(self.path_db)
  34. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool,
  35. flag_non_verbose: bool):
  36. """
  37. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  38. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  39. statistics are gathered directly from the existing database.
  40. :param flag_write_file: Indicates whether the statistics should be written addiotionally into a text file (True)
  41. or not (False)
  42. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  43. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  44. :param flag_non_verbose: Indicates whether certain prints should be made or not, to reduce terminal clutter
  45. """
  46. # Load pcap and get loading time
  47. time_start = time.clock()
  48. # Inform user about recalculation of statistics and its reason
  49. if flag_recalculate_stats:
  50. print("Flag -r/--recalculate found. Recalculating statistics.")
  51. # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
  52. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats or self.stats_db.get_db_outdated():
  53. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests))
  54. self.pcap_proc.collect_statistics()
  55. self.pcap_proc.write_to_database(self.path_db)
  56. outstring_datasource = "by PCAP file processor."
  57. # only print summary of new db if -s flag not set
  58. if not flag_print_statistics and not flag_non_verbose:
  59. self.stats_summary_new_db()
  60. else:
  61. outstring_datasource = "from statistics database."
  62. # Load statistics from database
  63. self.file_info = self.stats_db.get_file_info()
  64. time_end = time.clock()
  65. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  66. # Write statistics if param -e/--export provided
  67. if flag_write_file:
  68. self.write_statistics_to_file()
  69. # Print statistics if param -s/--statistics provided
  70. if flag_print_statistics:
  71. self.print_statistics()
  72. def get_file_information(self):
  73. """
  74. Returns a list of tuples, each containing a information of the file.
  75. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  76. """
  77. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  78. pdu_share = pdu_count / self.get_packet_count() * 100
  79. last_pdu_timestamp = self.process_db_query(
  80. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  81. return [("Pcap file path", self.pcap_filepath),
  82. ("Total packet count", self.get_packet_count(), "packets"),
  83. ("Recognized packets", self.get_packet_count() - pdu_count, "packets"),
  84. ("Unrecognized packets", pdu_count, "PDUs"),
  85. ("% Recognized packets", 100 - pdu_share, "%"),
  86. ("% Unrecognized packets", pdu_share, "%"),
  87. ("Last unknown PDU", last_pdu_timestamp),
  88. ("Capture duration", self.get_capture_duration(), "seconds"),
  89. ("Capture start", "\t" + str(self.get_pcap_timestamp_start())),
  90. ("Capture end", "\t" + str(self.get_pcap_timestamp_end()))]
  91. def get_general_file_statistics(self):
  92. """
  93. Returns a list of tuples, each containing a file statistic.
  94. :return: a list of tuples, each consisting of (description, value, unit).
  95. """
  96. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  97. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  98. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  99. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  100. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  101. @staticmethod
  102. def write_list(desc_val_unit_list, func, line_ending="\n"):
  103. """
  104. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three
  105. values and applies the function func on this string.
  106. Before generating the string, it identifies text containing a float number, casts the string to a
  107. float and rounds the value to two decimal digits.
  108. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  109. :param func: The function to be applied to each generated string
  110. :param line_ending: The formatting string to be applied at the end of each string
  111. """
  112. for entry in desc_val_unit_list:
  113. # Convert text containing float into float
  114. (description, value) = entry[0:2]
  115. if isinstance(value, str) and "." in value:
  116. try:
  117. value = float(value)
  118. except ValueError:
  119. pass # do nothing -> value was not a float
  120. # round float
  121. if isinstance(value, float):
  122. value = round(value, 4)
  123. # write into file
  124. if len(entry) == 3:
  125. unit = entry[2]
  126. func(description + ":\t" + str(value) + " " + unit + line_ending)
  127. else:
  128. func(description + ":\t" + str(value) + line_ending)
  129. def print_statistics(self):
  130. """
  131. Prints the basic file statistics to the terminal.
  132. """
  133. print("\nPCAP FILE INFORMATION ------------------------------")
  134. Statistics.write_list(self.get_file_information(), print, "")
  135. print("\nGENERAL FILE STATISTICS ----------------------------")
  136. Statistics.write_list(self.get_general_file_statistics(), print, "")
  137. print("\n")
  138. @staticmethod
  139. def calculate_entropy(frequency: list, normalized: bool = False):
  140. """
  141. Calculates entropy and normalized entropy of list of elements that have specific frequency
  142. :param frequency: The frequency of the elements.
  143. :param normalized: Calculate normalized entropy
  144. :return: entropy or (entropy, normalized entropy)
  145. """
  146. entropy, normalized_ent, n = 0, 0, 0
  147. sum_freq = sum(frequency)
  148. for i, x in enumerate(frequency):
  149. p_x = float(frequency[i] / sum_freq)
  150. if p_x > 0:
  151. n += 1
  152. entropy += - p_x * log(p_x, 2)
  153. if normalized:
  154. if log(n) > 0:
  155. normalized_ent = entropy / log(n, 2)
  156. return entropy, normalized_ent
  157. else:
  158. return entropy
  159. def calculate_complement_packet_rates(self, pps):
  160. """
  161. Calculates the complement packet rates of the background traffic packet rates for each interval.
  162. Then normalize it to maximum boundary, which is the input parameter pps
  163. :return: normalized packet rates for each time interval.
  164. """
  165. result = self.process_db_query(
  166. "SELECT lastPktTimestamp,pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  167. # print(result)
  168. bg_interval_pps = []
  169. complement_interval_pps = []
  170. intervals_sum = 0
  171. if result:
  172. # Get the interval in seconds
  173. for i, row in enumerate(result):
  174. if i < len(result) - 1:
  175. intervals_sum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
  176. interval = intervals_sum / (len(result) - 1)
  177. # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
  178. for row in result:
  179. bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
  180. # Find max PPS
  181. max_pps = max(bg_interval_pps, key=itemgetter(1))[1]
  182. for row in bg_interval_pps:
  183. complement_interval_pps.append((row[0], int(pps * (max_pps - row[1]) / max_pps)))
  184. return complement_interval_pps
  185. def get_tests_statistics(self):
  186. """
  187. Writes the calculated basic defects tests statistics into a file.
  188. """
  189. # self.stats_db.process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
  190. def count_frequncy(values_list):
  191. values, freq_output = [], []
  192. for x in values_list:
  193. if x in values:
  194. freq_output[values.index(x)] += 1
  195. else:
  196. values.append(x)
  197. freq_output.append(1)
  198. return values, freq_output
  199. # Payload Tests
  200. sum_payload_count = self.stats_db.process_user_defined_query("SELECT sum(payloadCount) FROM "
  201. "interval_statistics")
  202. pkt_count = self.stats_db.process_user_defined_query("SELECT packetCount FROM file_statistics")
  203. if sum_payload_count and pkt_count:
  204. payload_ratio = 0
  205. if pkt_count[0][0] != 0:
  206. payload_ratio = float(sum_payload_count[0][0] / pkt_count[0][0] * 100)
  207. else:
  208. payload_ratio = -1
  209. # TCP checksum Tests
  210. incorrect_checksum_count = self.stats_db.process_user_defined_query(
  211. "SELECT sum(incorrectTCPChecksumCount) FROM interval_statistics")
  212. correct_checksum_count = self.stats_db.process_user_defined_query(
  213. "SELECT avg(correctTCPChecksumCount) FROM interval_statistics")
  214. if incorrect_checksum_count and correct_checksum_count:
  215. incorrect_checksum_ratio = 0
  216. if (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) != 0:
  217. incorrect_checksum_ratio = float(incorrect_checksum_count[0][0] /
  218. (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) * 100)
  219. else:
  220. incorrect_checksum_ratio = -1
  221. # IP Src & Dst Tests
  222. result = self.stats_db.process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
  223. data, src_frequency, dst_frequency = [], [], []
  224. if result:
  225. for row in result:
  226. src_frequency.append(row[1])
  227. dst_frequency.append(row[2])
  228. ip_src_entropy, ip_src_norm_entropy = self.calculate_entropy(src_frequency, True)
  229. ip_dst_entropy, ip_dst_norm_entropy = self.calculate_entropy(dst_frequency, True)
  230. new_ip_count = self.stats_db.process_user_defined_query("SELECT newIPCount FROM interval_statistics")
  231. ip_novels_per_interval, ip_novels_per_interval_frequency = count_frequncy(new_ip_count)
  232. ip_novelty_dist_entropy = self.calculate_entropy(ip_novels_per_interval_frequency)
  233. # Ports Tests
  234. port0_count = self.stats_db.process_user_defined_query(
  235. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
  236. if not port0_count[0][0]:
  237. port0_count = 0
  238. else:
  239. port0_count = port0_count[0][0]
  240. # FIXME: could be extended
  241. reserved_port_count = self.stats_db.process_user_defined_query(
  242. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")
  243. if not reserved_port_count[0][0]:
  244. reserved_port_count = 0
  245. else:
  246. reserved_port_count = reserved_port_count[0][0]
  247. # TTL Tests
  248. result = self.stats_db.process_user_defined_query(
  249. "SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  250. data, frequency = [], []
  251. for row in result:
  252. frequency.append(row[1])
  253. ttl_entropy, ttl_norm_entropy = self.calculate_entropy(frequency, True)
  254. new_ttl_count = self.stats_db.process_user_defined_query("SELECT newTTLCount FROM interval_statistics")
  255. ttl_novels_per_interval, ttl_novels_per_interval_frequency = count_frequncy(new_ttl_count)
  256. ttl_novelty_dist_entropy = self.calculate_entropy(ttl_novels_per_interval_frequency)
  257. # Window Size Tests
  258. result = self.stats_db.process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
  259. data, frequency = [], []
  260. for row in result:
  261. frequency.append(row[1])
  262. win_entropy, win_norm_entropy = self.calculate_entropy(frequency, True)
  263. new_win_size_count = self.stats_db.process_user_defined_query("SELECT newWinSizeCount FROM interval_statistics")
  264. win_novels_per_interval, win_novels_per_interval_frequency = count_frequncy(new_win_size_count)
  265. win_novelty_dist_entropy = self.calculate_entropy(win_novels_per_interval_frequency)
  266. # ToS Tests
  267. result = self.stats_db.process_user_defined_query(
  268. "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
  269. data, frequency = [], []
  270. for row in result:
  271. frequency.append(row[1])
  272. tos_entropy, tos_norm_entropy = self.calculate_entropy(frequency, True)
  273. new_tos_count = self.stats_db.process_user_defined_query("SELECT newToSCount FROM interval_statistics")
  274. tos_novels_per_interval, tos_novels_per_interval_frequency = count_frequncy(new_tos_count)
  275. tos_novelty_dist_entropy = self.calculate_entropy(tos_novels_per_interval_frequency)
  276. # MSS Tests
  277. result = self.stats_db.process_user_defined_query(
  278. "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  279. data, frequency = [], []
  280. for row in result:
  281. frequency.append(row[1])
  282. mss_entropy, mss_norm_entropy = self.calculate_entropy(frequency, True)
  283. new_mss_count = self.stats_db.process_user_defined_query("SELECT newMSSCount FROM interval_statistics")
  284. mss_novels_per_interval, mss_novels_per_interval_frequency = count_frequncy(new_mss_count)
  285. mss_novelty_dist_entropy = self.calculate_entropy(mss_novels_per_interval_frequency)
  286. result = self.stats_db.process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
  287. # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
  288. if not result[0][0]:
  289. result = 0
  290. else:
  291. result = result[0][0]
  292. big_mss = (result / sum(frequency)) * 100
  293. output = []
  294. if self.do_extra_tests:
  295. output = [("Payload ratio", payload_ratio, "%"),
  296. ("Incorrect TCP checksum ratio", incorrect_checksum_ratio, "%")]
  297. output = output + [("# IP addresses", sum([x[0] for x in new_ip_count]), ""),
  298. ("IP Src Entropy", ip_src_entropy, ""),
  299. ("IP Src Normalized Entropy", ip_src_norm_entropy, ""),
  300. ("IP Dst Entropy", ip_dst_entropy, ""),
  301. ("IP Dst Normalized Entropy", ip_dst_norm_entropy, ""),
  302. ("IP Novelty Distribution Entropy", ip_novelty_dist_entropy, ""),
  303. ("# TTL values", sum([x[0] for x in new_ttl_count]), ""),
  304. ("TTL Entropy", ttl_entropy, ""),
  305. ("TTL Normalized Entropy", ttl_norm_entropy, ""),
  306. ("TTL Novelty Distribution Entropy", ttl_novelty_dist_entropy, ""),
  307. ("# WinSize values", sum([x[0] for x in new_win_size_count]), ""),
  308. ("WinSize Entropy", win_entropy, ""),
  309. ("WinSize Normalized Entropy", win_norm_entropy, ""),
  310. ("WinSize Novelty Distribution Entropy", win_novelty_dist_entropy, ""),
  311. ("# ToS values", sum([x[0] for x in new_tos_count]), ""),
  312. ("ToS Entropy", tos_entropy, ""),
  313. ("ToS Normalized Entropy", tos_norm_entropy, ""),
  314. ("ToS Novelty Distribution Entropy", tos_novelty_dist_entropy, ""),
  315. ("# MSS values", sum([x[0] for x in new_mss_count]), ""),
  316. ("MSS Entropy", mss_entropy, ""),
  317. ("MSS Normalized Entropy", mss_norm_entropy, ""),
  318. ("MSS Novelty Distribution Entropy", mss_novelty_dist_entropy, ""),
  319. ("======================", "", "")]
  320. # Reasoning the statistics values
  321. if self.do_extra_tests:
  322. if payload_ratio > 80:
  323. output.append(("WARNING: Too high payload ratio", payload_ratio, "%."))
  324. if payload_ratio < 30:
  325. output.append(("WARNING: Too low payload ratio", payload_ratio, "% (Injecting attacks that are carried "
  326. "out in the packet payloads is not "
  327. "recommmanded)."))
  328. if incorrect_checksum_ratio > 5:
  329. output.append(("WARNING: High incorrect TCP checksum ratio", incorrect_checksum_ratio, "%."))
  330. if ip_src_norm_entropy > 0.65:
  331. output.append(("WARNING: High IP source normalized entropy", ip_src_norm_entropy, "."))
  332. if ip_src_norm_entropy < 0.2:
  333. output.append(("WARNING: Low IP source normalized entropy", ip_src_norm_entropy, "."))
  334. if ip_dst_norm_entropy > 0.65:
  335. output.append(("WARNING: High IP destination normalized entropy", ip_dst_norm_entropy, "."))
  336. if ip_dst_norm_entropy < 0.2:
  337. output.append(("WARNING: Low IP destination normalized entropy", ip_dst_norm_entropy, "."))
  338. if ttl_norm_entropy > 0.65:
  339. output.append(("WARNING: High TTL normalized entropy", ttl_norm_entropy, "."))
  340. if ttl_norm_entropy < 0.2:
  341. output.append(("WARNING: Low TTL normalized entropy", ttl_norm_entropy, "."))
  342. if ttl_novelty_dist_entropy < 1:
  343. output.append(("WARNING: Too low TTL novelty distribution entropy", ttl_novelty_dist_entropy,
  344. "(The distribution of the novel TTL values is suspicious)."))
  345. if win_norm_entropy > 0.6:
  346. output.append(("WARNING: High Window Size normalized entropy", win_norm_entropy, "."))
  347. if win_norm_entropy < 0.1:
  348. output.append(("WARNING: Low Window Size normalized entropy", win_norm_entropy, "."))
  349. if win_novelty_dist_entropy < 4:
  350. output.append(("WARNING: Low Window Size novelty distribution entropy", win_novelty_dist_entropy,
  351. "(The distribution of the novel Window Size values is suspicious)."))
  352. if tos_norm_entropy > 0.4:
  353. output.append(("WARNING: High ToS normalized entropy", tos_norm_entropy, "."))
  354. if tos_norm_entropy < 0.1:
  355. output.append(("WARNING: Low ToS normalized entropy", tos_norm_entropy, "."))
  356. if tos_novelty_dist_entropy < 0.5:
  357. output.append(("WARNING: Low ToS novelty distribution entropy", tos_novelty_dist_entropy,
  358. "(The distribution of the novel ToS values is suspicious)."))
  359. if mss_norm_entropy > 0.4:
  360. output.append(("WARNING: High MSS normalized entropy", mss_norm_entropy, "."))
  361. if mss_norm_entropy < 0.1:
  362. output.append(("WARNING: Low MSS normalized entropy", mss_norm_entropy, "."))
  363. if mss_novelty_dist_entropy < 0.5:
  364. output.append(("WARNING: Low MSS novelty distribution entropy", mss_novelty_dist_entropy,
  365. "(The distribution of the novel MSS values is suspicious)."))
  366. if big_mss > 50:
  367. output.append(("WARNING: High ratio of MSS > 1460", big_mss, "% (High fragmentation rate in Ethernet)."))
  368. if port0_count > 0:
  369. output.append(("WARNING: Port number 0 is used in ", port0_count, "packets (awkward-looking port)."))
  370. if reserved_port_count > 0:
  371. output.append(("WARNING: Reserved port numbers are used in ", reserved_port_count,
  372. "packets (uncommonly-used ports)."))
  373. return output
  374. def write_statistics_to_file(self):
  375. """
  376. Writes the calculated basic statistics into a file.
  377. """
  378. def _write_header(title: str):
  379. """
  380. Writes the section header into the open file.
  381. :param title: The section title
  382. """
  383. target.write("====================== \n")
  384. target.write(title + " \n")
  385. target.write("====================== \n")
  386. target = open(self.pcap_filepath + ".stat", 'w')
  387. target.truncate()
  388. _write_header("PCAP file information")
  389. Statistics.write_list(self.get_file_information(), target.write)
  390. _write_header("General statistics")
  391. Statistics.write_list(self.get_general_file_statistics(), target.write)
  392. _write_header("Tests statistics")
  393. Statistics.write_list(self.get_tests_statistics(), target.write)
  394. target.close()
  395. def get_capture_duration(self):
  396. """
  397. :return: The duration of the capture in seconds
  398. """
  399. return self.file_info['captureDuration']
  400. def get_pcap_timestamp_start(self):
  401. """
  402. :return: The timestamp of the first packet in the PCAP file
  403. """
  404. return self.file_info['timestampFirstPacket']
  405. def get_pcap_timestamp_end(self):
  406. """
  407. :return: The timestamp of the last packet in the PCAP file
  408. """
  409. return self.file_info['timestampLastPacket']
  410. def get_pps_sent(self, ip_address: str):
  411. """
  412. Calculates the sent packets per seconds for a given IP address.
  413. :param ip_address: The IP address whose packets per second should be calculated
  414. :return: The sent packets per seconds for the given IP address
  415. """
  416. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  417. (ip_address,))
  418. capture_duration = float(self.get_capture_duration())
  419. return int(float(packets_sent) / capture_duration)
  420. def get_pps_received(self, ip_address: str):
  421. """
  422. Calculate the packets per second received for a given IP address.
  423. :param ip_address: The IP address used for the calculation
  424. :return: The number of packets per second received
  425. """
  426. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  427. False,
  428. (ip_address,))
  429. capture_duration = float(self.get_capture_duration())
  430. return int(float(packets_received) / capture_duration)
  431. def get_packet_count(self):
  432. """
  433. :return: The number of packets in the loaded PCAP file
  434. """
  435. return self.file_info['packetCount']
  436. def get_most_used_ip_address(self):
  437. """
  438. :return: The IP address/addresses with the highest sum of packets sent and received
  439. """
  440. return Util.handle_most_used_outputs(self.process_db_query("most_used(ipAddress)"))
  441. def get_ttl_distribution(self, ip_address: str):
  442. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ip_address + '"')
  443. result_dict = {key: value for (key, value) in result}
  444. return result_dict
  445. def get_mss_distribution(self, ip_address: str):
  446. result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ip_address + '"')
  447. result_dict = {key: value for (key, value) in result}
  448. return result_dict
  449. def get_win_distribution(self, ip_address: str):
  450. result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ip_address + '"')
  451. result_dict = {key: value for (key, value) in result}
  452. return result_dict
  453. def get_tos_distribution(self, ip_address: str):
  454. result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ip_address + '"')
  455. result_dict = {key: value for (key, value) in result}
  456. return result_dict
  457. def get_ip_address_count(self):
  458. return self.process_db_query("SELECT COUNT(*) FROM ip_statistics")
  459. def get_ip_addresses(self):
  460. return self.process_db_query("SELECT ipAddress FROM ip_statistics")
  461. def get_random_ip_address(self, count: int = 1):
  462. """
  463. :param count: The number of IP addreses to return
  464. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of
  465. randomly chosen IP addresses
  466. """
  467. ip_address_list = self.process_db_query("all(ipAddress)")
  468. if count == 1:
  469. return random.choice(ip_address_list)
  470. else:
  471. result_list = []
  472. for i in range(0, count):
  473. random_ip = random.choice(ip_address_list)
  474. result_list.append(random_ip)
  475. ip_address_list.remove(random_ip)
  476. return result_list
  477. def get_ip_address_from_mac(self, mac_address: str):
  478. """
  479. :param mac_address: the MAC address of which the IP shall be returned, if existing in DB
  480. :return: the IP address used in the dataset by a given MAC address
  481. """
  482. return self.process_db_query('ipAddress(macAddress=' + mac_address + ")")
  483. def get_mac_address(self, ip_address: str):
  484. """
  485. :return: The MAC address used in the dataset for the given IP address.
  486. """
  487. return self.process_db_query('macAddress(ipAddress=' + ip_address + ")")
  488. def get_most_used_mss(self, ip_address: str):
  489. """
  490. :param ip_address: The IP address whose used MSS should be determined
  491. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  492. then None is returned
  493. """
  494. mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ip_address +
  495. '" AND mssCount == (SELECT MAX(mssCount) from tcp_mss WHERE ipAddress="'
  496. + ip_address + '")')
  497. if isinstance(mss_value, int):
  498. return mss_value
  499. elif isinstance(mss_value, list):
  500. if len(mss_value) == 0:
  501. return None
  502. else:
  503. mss_value.sort()
  504. return mss_value[0]
  505. else:
  506. return None
  507. def get_most_used_ttl(self, ip_address: str):
  508. """
  509. :param ip_address: The IP address whose used TTL should be determined
  510. :return: The TTL value used by the IP address, or if the IP addresses never specified a TTL,
  511. then None is returned
  512. """
  513. ttl_value = self.process_db_query('SELECT ttlValue from ip_ttl WHERE ipAddress="' + ip_address +
  514. '" AND ttlCount == (SELECT MAX(ttlCount) from ip_ttl WHERE ipAddress="'
  515. + ip_address + '")')
  516. if isinstance(ttl_value, int):
  517. return ttl_value
  518. elif isinstance(ttl_value, list):
  519. if len(ttl_value) == 0:
  520. return None
  521. else:
  522. ttl_value.sort()
  523. return ttl_value[0]
  524. else:
  525. return None
  526. def get_avg_delay_local_ext(self):
  527. """
  528. Calculates the average delay of a packet for external and local communication, based on the tcp handshakes
  529. :return: tuple consisting of avg delay for local and external communication, (local, external)
  530. """
  531. conv_delays = self.stats_db.process_user_defined_query("SELECT ipAddressA, ipAddressB, avgDelay FROM conv_statistics")
  532. if(conv_delays):
  533. external_conv = []
  534. local_conv = []
  535. for conv in conv_delays:
  536. IPA = IPAddress.parse(conv[0])
  537. IPB = IPAddress.parse(conv[1])
  538. #split into local and external conversations
  539. if(not IPA.is_private() or not IPB.is_private()):
  540. external_conv.append(conv)
  541. else:
  542. local_conv.append(conv)
  543. # calculate avg local and external delay by summing up the respective delays and dividing them by the number of conversations
  544. avg_delay_external = 0.0
  545. avg_delay_local = 0.0
  546. default_ext = False
  547. default_local = False
  548. if(local_conv):
  549. for conv in local_conv:
  550. avg_delay_local += conv[2]
  551. avg_delay_local = (avg_delay_local/len(local_conv)) * 0.001 #ms
  552. else:
  553. # no local conversations in statistics found
  554. avg_delay_local = 0.055
  555. default_local = True
  556. if(external_conv):
  557. for conv in external_conv:
  558. avg_delay_external += conv[2]
  559. avg_delay_external = (avg_delay_external/len(external_conv)) * 0.001 #ms
  560. else:
  561. # no external conversations in statistics found
  562. avg_delay_external = 0.09
  563. default_ext = True
  564. else:
  565. #if no statistics were found, use these numbers
  566. avg_delay_external = 0.09
  567. avg_delay_local = 0.055
  568. default_ext = True
  569. default_local = True
  570. # check whether delay numbers are consistent
  571. if avg_delay_local > avg_delay_external:
  572. avg_delay_external = avg_delay_local*1.2
  573. # print information, that (default) values are used, that are not collected from the Input PCAP
  574. if default_ext or default_local:
  575. if default_ext and default_local:
  576. print("Warning: Could not collect average delays for local or external communication, using following values:")
  577. elif default_ext:
  578. print("Warning: Could not collect average delays for external communication, using following values:")
  579. elif default_local:
  580. print("Warning: Could not collect average delays for local communication, using following values:")
  581. print("Avg delay of external communication: {0}s, Avg delay of local communication: {1}s".format(avg_delay_external, avg_delay_local))
  582. return avg_delay_local, avg_delay_external
  583. def get_filtered_degree(self, degree_type: str):
  584. """
  585. gets the desired type of degree statistics and filters IPs with degree value zero
  586. :param degree_type: the desired type of degrees, one of the following: inDegree, outDegree, overallDegree
  587. :return: the filtered degrees
  588. """
  589. degrees_raw = self.stats_db.process_user_defined_query(
  590. "SELECT ipAddress, %s FROM ip_degrees" % degree_type)
  591. degrees = []
  592. if(degrees_raw):
  593. for deg in degrees_raw:
  594. if int(deg[1]) > 0:
  595. degrees.append(deg)
  596. return degrees
  597. def get_rnd_win_size(self, pkts_num):
  598. """
  599. :param pkts_num: maximum number of window sizes, that should be returned
  600. :return: A list of randomly chosen window sizes with given length.
  601. """
  602. sql_return = self.process_db_query("SELECT DISTINCT winSize FROM tcp_win ORDER BY winsize ASC;")
  603. if not isinstance(sql_return, list):
  604. return [sql_return]
  605. result = []
  606. for i in range(0, min(pkts_num, len(sql_return))):
  607. result.append(random.choice(sql_return))
  608. sql_return.remove(result[i])
  609. return result
  610. def get_statistics_database(self):
  611. """
  612. :return: A reference to the statistics database object
  613. """
  614. return self.stats_db
  615. def process_db_query(self, query_string_in: str, print_results: bool = False):
  616. """
  617. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  618. query.
  619. :param query_string_in: The query to be processed
  620. :param print_results: Indicates whether the results should be printed to terminal
  621. :return: The result of the query
  622. """
  623. return self.stats_db.process_db_query(query_string_in, print_results)
  624. def is_query(self, value: str):
  625. """
  626. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  627. :param value: The string to be checked
  628. :return: True if the string is recognized as a query, otherwise False.
  629. """
  630. if not isinstance(value, str):
  631. return False
  632. else:
  633. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  634. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  635. @staticmethod
  636. def calculate_standard_deviation(lst):
  637. """
  638. Calculates the standard deviation of a list of numbers.
  639. :param lst: The list of numbers to calculate its SD.
  640. """
  641. num_items = len(lst)
  642. mean = sum(lst) / num_items
  643. differences = [x - mean for x in lst]
  644. sq_differences = [d ** 2 for d in differences]
  645. ssd = sum(sq_differences)
  646. variance = ssd / num_items
  647. sd = sqrt(variance)
  648. return sd
  649. def plot_statistics(self, entropy: int, file_format: str = 'pdf'): # 'png'
  650. """
  651. Plots the statistics associated with the dataset.
  652. :param entropy: the statistics entropy
  653. :param file_format: The format to be used to save the statistics diagrams.
  654. """
  655. def plot_distribution(query_output, title, x_label, y_label, file_ending: str):
  656. plt.gcf().clear()
  657. graphx, graphy = [], []
  658. for row in query_output:
  659. graphx.append(row[0])
  660. graphy.append(row[1])
  661. plt.autoscale(enable=True, axis='both')
  662. plt.title(title)
  663. plt.xlabel(x_label)
  664. plt.ylabel(y_label)
  665. width = 0.1
  666. plt.xlim([0, (max(graphx) * 1.1)])
  667. plt.grid(True)
  668. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  669. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  670. plt.savefig(out, dpi=500)
  671. return out
  672. def plot_ttl(file_ending: str):
  673. query_output = self.stats_db.process_user_defined_query(
  674. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  675. title = "TTL Distribution"
  676. x_label = "TTL Value"
  677. y_label = "Number of Packets"
  678. if query_output:
  679. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  680. def plot_mss(file_ending: str):
  681. query_output = self.stats_db.process_user_defined_query(
  682. "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  683. title = "MSS Distribution"
  684. x_label = "MSS Value"
  685. y_label = "Number of Packets"
  686. if query_output:
  687. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  688. def plot_win(file_ending: str):
  689. query_output = self.stats_db.process_user_defined_query(
  690. "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
  691. title = "Window Size Distribution"
  692. x_label = "Window Size"
  693. y_label = "Number of Packets"
  694. if query_output:
  695. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  696. def plot_protocol(file_ending: str):
  697. plt.gcf().clear()
  698. result = self.stats_db.process_user_defined_query(
  699. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  700. if result:
  701. graphx, graphy = [], []
  702. for row in result:
  703. graphx.append(row[0])
  704. graphy.append(row[1])
  705. plt.autoscale(enable=True, axis='both')
  706. plt.title("Protocols Distribution")
  707. plt.xlabel('Protocols')
  708. plt.ylabel('Number of Packets')
  709. width = 0.1
  710. plt.xlim([0, len(graphx)])
  711. plt.grid(True)
  712. # Protocols' names on x-axis
  713. x = range(0, len(graphx))
  714. my_xticks = graphx
  715. plt.xticks(x, my_xticks)
  716. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  717. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  718. plt.savefig(out, dpi=500)
  719. return out
  720. else:
  721. print("Error plot protocol: No protocol values found!")
  722. def plot_port(file_ending: str):
  723. plt.gcf().clear()
  724. result = self.stats_db.process_user_defined_query(
  725. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  726. graphx, graphy = [], []
  727. for row in result:
  728. graphx.append(row[0])
  729. graphy.append(row[1])
  730. plt.autoscale(enable=True, axis='both')
  731. plt.title("Ports Distribution")
  732. plt.xlabel('Ports Numbers')
  733. plt.ylabel('Number of Packets')
  734. width = 0.1
  735. plt.xlim([0, max(graphx)])
  736. plt.grid(True)
  737. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  738. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  739. plt.savefig(out, dpi=500)
  740. return out
  741. # This distribution is not drawable for big datasets
  742. def plot_ip_src(file_ending: str):
  743. plt.gcf().clear()
  744. result = self.stats_db.process_user_defined_query(
  745. "SELECT ipAddress, pktsSent FROM ip_statistics")
  746. graphx, graphy = [], []
  747. for row in result:
  748. graphx.append(row[0])
  749. graphy.append(row[1])
  750. plt.autoscale(enable=True, axis='both')
  751. plt.title("Source IP Distribution")
  752. plt.xlabel('Source IP')
  753. plt.ylabel('Number of Packets')
  754. width = 0.1
  755. plt.xlim([0, len(graphx)])
  756. plt.grid(True)
  757. # IPs on x-axis
  758. x = range(0, len(graphx))
  759. my_xticks = graphx
  760. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  761. plt.tight_layout()
  762. # limit the number of xticks
  763. plt.locator_params(axis='x', nbins=20)
  764. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  765. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  766. plt.savefig(out, dpi=500)
  767. return out
  768. # This distribution is not drawable for big datasets
  769. def plot_ip_dst(file_ending: str):
  770. plt.gcf().clear()
  771. result = self.stats_db.process_user_defined_query(
  772. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  773. graphx, graphy = [], []
  774. for row in result:
  775. graphx.append(row[0])
  776. graphy.append(row[1])
  777. plt.autoscale(enable=True, axis='both')
  778. plt.title("Destination IP Distribution")
  779. plt.xlabel('Destination IP')
  780. plt.ylabel('Number of Packets')
  781. width = 0.1
  782. plt.xlim([0, len(graphx)])
  783. plt.grid(True)
  784. # IPs on x-axis
  785. x = range(0, len(graphx))
  786. my_xticks = graphx
  787. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  788. plt.tight_layout()
  789. # limit the number of xticks
  790. plt.locator_params(axis='x', nbins=20)
  791. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  792. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  793. plt.savefig(out, dpi=500)
  794. return out
  795. def plot_interval_statistics(query_output, title, x_label, y_label, file_ending: str):
  796. plt.gcf().clear()
  797. graphx, graphy = [], []
  798. for row in query_output:
  799. graphx.append(row[0])
  800. graphy.append(row[1])
  801. plt.autoscale(enable=True, axis='both')
  802. plt.title(title)
  803. plt.xlabel(x_label)
  804. plt.ylabel(y_label)
  805. width = 0.5
  806. plt.xlim([0, len(graphx)])
  807. plt.grid(True)
  808. # timestamp on x-axis
  809. x = range(0, len(graphx))
  810. # limit the number of xticks
  811. plt.locator_params(axis='x', nbins=20)
  812. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  813. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  814. plt.savefig(out, dpi=500)
  815. return out
  816. def plot_interval_pkt_count(file_ending: str):
  817. query_output = self.stats_db.process_user_defined_query(
  818. "SELECT lastPktTimestamp, pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  819. title = "Packet Rate"
  820. x_label = "Time Interval"
  821. y_label = "Number of Packets"
  822. if query_output:
  823. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  824. def plot_interval_ip_src_ent(file_ending: str):
  825. query_output = self.stats_db.process_user_defined_query(
  826. "SELECT lastPktTimestamp, ipSrcEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  827. title = "Source IP Entropy"
  828. x_label = "Time Interval"
  829. y_label = "Entropy"
  830. if query_output:
  831. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  832. def plot_interval_ip_dst_ent(file_ending: str):
  833. query_output = self.stats_db.process_user_defined_query(
  834. "SELECT lastPktTimestamp, ipDstEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  835. title = "Destination IP Entropy"
  836. x_label = "Time Interval"
  837. y_label = "Entropy"
  838. if query_output:
  839. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  840. def plot_interval_new_ip(file_ending: str):
  841. query_output = self.stats_db.process_user_defined_query(
  842. "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
  843. title = "IP Novelty Distribution"
  844. x_label = "Time Interval"
  845. y_label = "Novel values count"
  846. if query_output:
  847. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  848. def plot_interval_new_port(file_ending: str):
  849. query_output = self.stats_db.process_user_defined_query(
  850. "SELECT lastPktTimestamp, newPortCount FROM interval_statistics ORDER BY lastPktTimestamp")
  851. title = "Port Novelty Distribution"
  852. x_label = "Time Interval"
  853. y_label = "Novel values count"
  854. if query_output:
  855. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  856. def plot_interval_new_ttl(file_ending: str):
  857. query_output = self.stats_db.process_user_defined_query(
  858. "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
  859. title = "TTL Novelty Distribution"
  860. x_label = "Time Interval"
  861. y_label = "Novel values count"
  862. if query_output:
  863. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  864. def plot_interval_new_tos(file_ending: str):
  865. query_output = self.stats_db.process_user_defined_query(
  866. "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  867. title = "ToS Novelty Distribution"
  868. x_label = "Time Interval"
  869. y_label = "Novel values count"
  870. if query_output:
  871. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  872. def plot_interval_new_win_size(file_ending: str):
  873. query_output = self.stats_db.process_user_defined_query(
  874. "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
  875. title = "Window Size Novelty Distribution"
  876. x_label = "Time Interval"
  877. y_label = "Novel values count"
  878. if query_output:
  879. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  880. def plot_interval_new_mss(file_ending: str):
  881. query_output = self.stats_db.process_user_defined_query(
  882. "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  883. title = "MSS Novelty Distribution"
  884. x_label = "Time Interval"
  885. y_label = "Novel values count"
  886. if query_output:
  887. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  888. def plot_interval_ip_dst_cum_ent(file_ending: str):
  889. plt.gcf().clear()
  890. result = self.stats_db.process_user_defined_query(
  891. "SELECT lastPktTimestamp, ipDstCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  892. graphx, graphy = [], []
  893. for row in result:
  894. graphx.append(row[0])
  895. graphy.append(row[1])
  896. # If entropy was not calculated do not plot the graph
  897. if graphy[0] != -1:
  898. plt.autoscale(enable=True, axis='both')
  899. plt.title("Destination IP Cumulative Entropy")
  900. # plt.xlabel('Timestamp')
  901. plt.xlabel('Time Interval')
  902. plt.ylabel('Entropy')
  903. plt.xlim([0, len(graphx)])
  904. plt.grid(True)
  905. # timestamp on x-axis
  906. x = range(0, len(graphx))
  907. # my_xticks = graphx
  908. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  909. # plt.tight_layout()
  910. # limit the number of xticks
  911. plt.locator_params(axis='x', nbins=20)
  912. plt.plot(x, graphy, 'r')
  913. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-cum-ent' + file_ending)
  914. plt.savefig(out, dpi=500)
  915. return out
  916. def plot_interval_ip_src_cum_ent(file_ending: str):
  917. plt.gcf().clear()
  918. result = self.stats_db.process_user_defined_query(
  919. "SELECT lastPktTimestamp, ipSrcCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  920. graphx, graphy = [], []
  921. for row in result:
  922. graphx.append(row[0])
  923. graphy.append(row[1])
  924. # If entropy was not calculated do not plot the graph
  925. if graphy[0] != -1:
  926. plt.autoscale(enable=True, axis='both')
  927. plt.title("Source IP Cumulative Entropy")
  928. # plt.xlabel('Timestamp')
  929. plt.xlabel('Time Interval')
  930. plt.ylabel('Entropy')
  931. plt.xlim([0, len(graphx)])
  932. plt.grid(True)
  933. # timestamp on x-axis
  934. x = range(0, len(graphx))
  935. # my_xticks = graphx
  936. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  937. # plt.tight_layout()
  938. # limit the number of xticks
  939. plt.locator_params(axis='x', nbins=20)
  940. plt.plot(x, graphy, 'r')
  941. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-cum-ent' + file_ending)
  942. plt.savefig(out, dpi=500)
  943. return out
  944. def plot_in_degree(file_ending: str):
  945. """
  946. Creates a Plot, visualizing the in-degree for every IP Address
  947. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  948. :return: A filepath to the file containing the created plot
  949. """
  950. plt.gcf().clear()
  951. # retrieve data
  952. in_degree = self.get_filtered_degree("inDegree")
  953. graphx, graphy = [], []
  954. for entry in in_degree:
  955. # degree values
  956. graphx.append(entry[1])
  957. # IP labels
  958. graphy.append(entry[0])
  959. # set labels
  960. plt.title("Indegree per IP Address")
  961. plt.ylabel('IpAddress')
  962. plt.xlabel('Indegree')
  963. #set width of the bars
  964. width = 0.3
  965. # set scalings
  966. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  967. #set limits of the axis
  968. plt.ylim([0, len(graphy)])
  969. plt.xlim([0, max(graphx) + 10])
  970. # display numbers at each bar
  971. for i, v in enumerate(graphx):
  972. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  973. # display grid for better visuals
  974. plt.grid(True)
  975. # plot the bar
  976. labels = graphy
  977. graphy = list(range(len(graphx)))
  978. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  979. plt.yticks(graphy, labels)
  980. out = self.pcap_filepath.replace('.pcap', '_plot-In Degree of an IP' + file_ending)
  981. plt.tight_layout()
  982. plt.savefig(out,dpi=500)
  983. return out
  984. def plot_out_degree(file_ending: str):
  985. """
  986. Creates a Plot, visualizing the out-degree for every IP Address
  987. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  988. :return: A filepath to the file containing the created plot
  989. """
  990. plt.gcf().clear()
  991. # retrieve data
  992. out_degree = self.get_filtered_degree("outDegree")
  993. graphx, graphy = [], []
  994. for entry in out_degree:
  995. # degree values
  996. graphx.append(entry[1])
  997. # IP labels
  998. graphy.append(entry[0])
  999. # set labels
  1000. plt.title("Outdegree per IP Address")
  1001. plt.ylabel('IpAddress')
  1002. plt.xlabel('Outdegree')
  1003. #set width of the bars
  1004. width = 0.3
  1005. # set scalings
  1006. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  1007. #set limits of the axis
  1008. plt.ylim([0, len(graphy)])
  1009. plt.xlim([0, max(graphx) + 10])
  1010. # display numbers at each bar
  1011. for i, v in enumerate(graphx):
  1012. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  1013. # display grid for better visuals
  1014. plt.grid(True)
  1015. # plot the bar
  1016. labels = graphy
  1017. graphy = list(range(len(graphx)))
  1018. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  1019. plt.yticks(graphy, labels)
  1020. out = self.pcap_filepath.replace('.pcap', '_plot-Out Degree of an IP' + file_ending)
  1021. plt.tight_layout()
  1022. plt.savefig(out,dpi=500)
  1023. return out
  1024. def plot_overall_degree(file_ending: str):
  1025. """
  1026. Creates a Plot, visualizing the overall-degree for every IP Address
  1027. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  1028. :return: A filepath to the file containing the created plot
  1029. """
  1030. plt.gcf().clear()
  1031. # retrieve data
  1032. overall_degree = self.get_filtered_degree("overallDegree")
  1033. graphx, graphy = [], []
  1034. for entry in overall_degree:
  1035. # degree values
  1036. graphx.append(entry[1])
  1037. # IP labels
  1038. graphy.append(entry[0])
  1039. # set labels
  1040. plt.title("Overalldegree per IP Address")
  1041. plt.ylabel('IpAddress')
  1042. plt.xlabel('Overalldegree')
  1043. #set width of the bars
  1044. width = 0.3
  1045. # set scalings
  1046. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  1047. #set limits of the axis
  1048. plt.ylim([0, len(graphy)])
  1049. plt.xlim([0, max(graphx) + 10])
  1050. # display numbers at each bar
  1051. for i, v in enumerate(graphx):
  1052. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  1053. # display grid for better visuals
  1054. plt.grid(True)
  1055. # plot the bar
  1056. labels = graphy
  1057. graphy = list(range(len(graphx)))
  1058. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  1059. plt.yticks(graphy, labels)
  1060. out = self.pcap_filepath.replace('.pcap', '_plot-Overall Degree of an IP' + file_ending)
  1061. plt.tight_layout()
  1062. plt.savefig(out,dpi=500)
  1063. return out
  1064. def plot_big_conv_ext_stat(attr:str, title:str, xlabel:str, suffix:str):
  1065. """
  1066. Plots the desired statistc per connection as horizontal bar plot.
  1067. Included are 'half-open' connections, where only one packet is exchanged.
  1068. The given statistics table has to have at least the attributes 'ipAddressA', 'portA', 'ipAddressB',
  1069. 'portB' and the specified additional attribute.
  1070. Note: there may be cutoff/scaling problems within the plot if there is too little data.
  1071. :param attr: The desired statistic, named with respect to its attribute in the given statistics table
  1072. :param table: The statistics table
  1073. :param title: The title of the created plot
  1074. :param xlabel: The name of the x-axis of the created plot
  1075. :param suffix: The suffix of the created file, including file extension
  1076. :return: A filepath to the file containing the created plot
  1077. """
  1078. plt.gcf().clear()
  1079. result = self.stats_db.process_user_defined_query(
  1080. "SELECT ipAddressA, portA, ipAddressB, portB, %s FROM conv_statistics_extended" % attr)
  1081. if (result):
  1082. graphy, graphx = [], []
  1083. # plot data in descending order
  1084. result = sorted(result, key=lambda row: row[4])
  1085. # compute plot data
  1086. for i, row in enumerate(result):
  1087. addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
  1088. # adjust the justification of strings to improve appearance
  1089. len_max = max(len(addr1), len(addr2))
  1090. addr1 = addr1.ljust(len_max)
  1091. addr2 = addr2.ljust(len_max)
  1092. # add plot data
  1093. graphy.append("%s\n%s" % (addr1, addr2))
  1094. graphx.append(row[4])
  1095. # have x axis and its label appear at the top (instead of bottom)
  1096. fig, ax = plt.subplots()
  1097. ax.xaxis.tick_top()
  1098. ax.xaxis.set_label_position("top")
  1099. # compute plot height in inches for scaling the plot
  1100. dist_mult_height, dist_mult_width = 0.55, 0.07 # these values turned out to work well
  1101. # use static scale along the conversation axis, if there are too little entries to use dynamic scaling numbers
  1102. if len(graphy) < 10:
  1103. plt_height = 7.5
  1104. # otherwise use the numbers above
  1105. else:
  1106. plt_height = len(graphy) * dist_mult_height
  1107. # use static scale along the x axis, if the x values are all 0
  1108. if max(graphx) < 200:
  1109. plt_width = 7.5 # 7.5 as static width worked well
  1110. if max(graphx) == 0:
  1111. ax.set_xlim(0, 10)
  1112. # otherwise use the numbers above
  1113. else:
  1114. plt_width = max(graphx) * dist_mult_width
  1115. title_distance = 1 + 0.012*52.8/plt_height # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
  1116. # if title would be cut off, set minimum width
  1117. min_width = len(title) * 0.15
  1118. if plt_width < min_width:
  1119. plt_width = min_width
  1120. plt.gcf().set_size_inches(plt_width, plt_height) # set plot size
  1121. # set additional plot parameters
  1122. plt.title(title, y=title_distance)
  1123. plt.xlabel(xlabel)
  1124. plt.ylabel('Connection')
  1125. width = 0.5
  1126. plt.grid(True)
  1127. plt.gca().margins(y=0) # removes the space between data and x-axis within the plot
  1128. # plot the above data, first use plain numbers as graphy to maintain sorting
  1129. plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=0.5, color='red', edgecolor='red')
  1130. # now change the y numbers to the respective address labels
  1131. plt.yticks(range(len(graphy)), graphy)
  1132. # try to use tight layout to cut off unnecessary space
  1133. try:
  1134. plt.tight_layout(pad=4)
  1135. except (ValueError, numpy.linalg.linalg.LinAlgError):
  1136. pass
  1137. # save created figure
  1138. out = self.pcap_filepath.replace('.pcap', suffix)
  1139. plt.savefig(out, dpi=500)
  1140. return out
  1141. def plot_packets_per_connection(file_ending: str):
  1142. """
  1143. Plots the total number of exchanged packets per connection.
  1144. :param file_ending: The file extension for the output file containing the plot
  1145. :return: A filepath to the file containing the created plot
  1146. """
  1147. title = 'Number of exchanged packets per connection'
  1148. suffix = '_plot-PktCount per Connection Distribution' + file_ending
  1149. # plot data and return outpath
  1150. return plot_big_conv_ext_stat("pktsCount", title, "Number of packets", suffix)
  1151. def plot_avg_pkts_per_comm_interval(file_ending: str):
  1152. """
  1153. Plots the average number of exchanged packets per communication interval for every connection.
  1154. :param file_ending: The file extension for the output file containing the plot
  1155. :return: A filepath to the file containing the created plot
  1156. """
  1157. title = 'Average number of exchanged packets per communication interval'
  1158. suffix = '_plot-Avg PktCount Communication Interval Distribution' + file_ending
  1159. # plot data and return outpath
  1160. return plot_big_conv_ext_stat("avgIntervalPktCount", title, "Number of packets", suffix)
  1161. def plot_avg_time_between_comm_interval(file_ending: str):
  1162. """
  1163. Plots the average time between the communication intervals of every connection.
  1164. :param file_ending: The file extension for the output file containing the plot
  1165. :return: A filepath to the file containing the created plot
  1166. """
  1167. title = 'Average time between communication intervals in seconds'
  1168. suffix = '_plot-Avg Time Between Communication Intervals Distribution' + file_ending
  1169. # plot data and return outpath
  1170. return plot_big_conv_ext_stat("avgTimeBetweenIntervals", title, 'Average time between intervals', suffix)
  1171. def plot_avg_comm_interval_time(file_ending: str):
  1172. """
  1173. Plots the average duration of a communication interval of every connection.
  1174. :param file_ending: The file extension for the output file containing the plot
  1175. :return: A filepath to the file containing the created plot
  1176. """
  1177. title = 'Average duration of a communication interval in seconds'
  1178. suffix = '_plot-Avg Duration Communication Interval Distribution' + file_ending
  1179. # plot data and return outpath
  1180. return plot_big_conv_ext_stat("avgIntervalTime", title, 'Average interval time', suffix)
  1181. def plot_total_comm_duration(file_ending: str):
  1182. """
  1183. Plots the total communication duration of every connection.
  1184. :param file_ending: The file extension for the output file containing the plot
  1185. :return: A filepath to the file containing the created plot
  1186. """
  1187. title = 'Total communication duration in seconds'
  1188. suffix = '_plot-Total Communication Duration Distribution' + file_ending
  1189. # plot data and return outpath
  1190. return plot_big_conv_ext_stat("totalConversationDuration", title, 'Duration', suffix)
  1191. ttl_out_path = plot_ttl('.' + file_format)
  1192. mss_out_path = plot_mss('.' + file_format)
  1193. win_out_path = plot_win('.' + file_format)
  1194. protocol_out_path = plot_protocol('.' + file_format)
  1195. plot_interval_pktCount = plot_interval_pkt_count('.' + file_format)
  1196. if entropy:
  1197. plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + file_format)
  1198. plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + file_format)
  1199. plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + file_format)
  1200. plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + file_format)
  1201. plot_interval_new_ip = plot_interval_new_ip('.' + file_format)
  1202. plot_interval_new_port = plot_interval_new_port('.' + file_format)
  1203. plot_interval_new_ttl = plot_interval_new_ttl('.' + file_format)
  1204. plot_interval_new_tos = plot_interval_new_tos('.' + file_format)
  1205. plot_interval_new_win_size = plot_interval_new_win_size('.' + file_format)
  1206. plot_interval_new_mss = plot_interval_new_mss('.' + file_format)
  1207. plot_out_degree = plot_out_degree('.' + file_format)
  1208. plot_in_degree = plot_in_degree('.' + file_format)
  1209. plot_overall_degree = plot_overall_degree('.' + file_format)
  1210. plot_packets_per_connection_out = plot_packets_per_connection('.' + file_format)
  1211. plot_avg_pkts_per_comm_interval_out = plot_avg_pkts_per_comm_interval('.' + file_format)
  1212. plot_avg_time_between_comm_interval_out = plot_avg_time_between_comm_interval('.' + file_format)
  1213. plot_avg_comm_interval_time_out = plot_avg_comm_interval_time("." + file_format)
  1214. plot_total_comm_duration_out = plot_total_comm_duration("." + file_format)
  1215. # Time consuming plot
  1216. # port_out_path = plot_port('.' + format)
  1217. # Not drawable for too many IPs
  1218. # ip_src_out_path = plot_ip_src('.' + format)
  1219. # ip_dst_out_path = plot_ip_dst('.' + format)
  1220. print("Saved plots in the input PCAP directory.")
  1221. def stats_summary_post_attack(self, added_packets):
  1222. """
  1223. Prints a summary of relevant statistics after an attack is injected
  1224. :param added_packets: sum of packets added by attacks, gets updated if more than one attack
  1225. :return: None
  1226. """
  1227. total_packet_count = self.get_packet_count() + added_packets
  1228. added_packets_share = added_packets / total_packet_count * 100
  1229. timespan = self.get_capture_duration()
  1230. summary = [("Total packet count", total_packet_count, "packets"),
  1231. ("Added packet count", added_packets, "packets"),
  1232. ("Share of added packets", added_packets_share, "%"),
  1233. ("Capture duration", timespan, "seconds")]
  1234. print("\nPOST INJECTION STATISTICS SUMMARY --------------------------")
  1235. self.write_list(summary, print, "")
  1236. print("------------------------------------------------------------")
  1237. def stats_summary_new_db(self):
  1238. """
  1239. Prints a summary of relevant statistics when a new db is created
  1240. :return: None
  1241. """
  1242. self.file_info = self.stats_db.get_file_info()
  1243. print("\nNew database has been generated, printing statistics summary... ")
  1244. total_packet_count = self.get_packet_count()
  1245. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  1246. pdu_share = pdu_count / total_packet_count * 100
  1247. last_pdu_timestamp = self.process_db_query(
  1248. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  1249. timespan = self.get_capture_duration()
  1250. summary = [("Total packet count", total_packet_count, "packets"),
  1251. ("Recognized packets", total_packet_count - pdu_count, "packets"),
  1252. ("Unrecognized packets", pdu_count, "PDUs"),
  1253. ("% Recognized packets", 100 - pdu_share, "%"),
  1254. ("% Unrecognized packets", pdu_share, "%"),
  1255. ("Last unknown PDU", last_pdu_timestamp),
  1256. ("Capture duration", timespan, "seconds")]
  1257. print("\nPCAP FILE STATISTICS SUMMARY ------------------------------")
  1258. self.write_list(summary, print, "")
  1259. print("------------------------------------------------------------")