Statistics.py 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269
  1. import os
  2. import random
  3. import time
  4. from math import sqrt, ceil, log
  5. from operator import itemgetter
  6. # TODO: double check this import
  7. # does it complain because libpcapreader is not a .py?
  8. import ID2TLib.libpcapreader as pr
  9. import matplotlib
  10. import Core.StatsDatabase as statsDB
  11. import ID2TLib.PcapFile as PcapFile
  12. import ID2TLib.Utility as Util
  13. matplotlib.use('Agg', force=True)
  14. import matplotlib.pyplot as plt
  15. class Statistics:
  16. def __init__(self, pcap_file: PcapFile.PcapFile):
  17. """
  18. Creates a new Statistics object.
  19. :param pcap_file: A reference to the PcapFile object
  20. """
  21. # Fields
  22. self.pcap_filepath = pcap_file.pcap_file_path
  23. self.pcap_proc = None
  24. self.do_extra_tests = False
  25. self.file_info = None
  26. # Create folder for statistics database if required
  27. self.path_db = pcap_file.get_db_path()
  28. path_dir = os.path.dirname(self.path_db)
  29. if not os.path.isdir(path_dir):
  30. os.makedirs(path_dir)
  31. # Class instances
  32. self.stats_db = statsDB.StatsDatabase(self.path_db)
  33. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool,
  34. flag_non_verbose: bool):
  35. """
  36. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  37. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  38. statistics are gathered directly from the existing database.
  39. :param flag_write_file: Indicates whether the statistics should be written additionally into a text file (True)
  40. or not (False)
  41. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  42. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  43. :param flag_non_verbose: Indicates whether certain prints should be made or not, to reduce terminal clutter
  44. """
  45. # Load pcap and get loading time
  46. time_start = time.clock()
  47. # Inform user about recalculation of statistics and its reason
  48. if flag_recalculate_stats:
  49. print("Flag -r/--recalculate found. Recalculating statistics.")
  50. # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
  51. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats or self.stats_db.get_db_outdated():
  52. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests))
  53. self.pcap_proc.collect_statistics()
  54. self.pcap_proc.write_to_database(self.path_db)
  55. outstring_datasource = "by PCAP file processor."
  56. # only print summary of new db if -s flag not set
  57. if not flag_print_statistics and not flag_non_verbose:
  58. self.stats_summary_new_db()
  59. else:
  60. outstring_datasource = "from statistics database."
  61. # Load statistics from database
  62. self.file_info = self.stats_db.get_file_info()
  63. time_end = time.clock()
  64. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  65. # Write statistics if param -e/--export provided
  66. if flag_write_file:
  67. self.write_statistics_to_file()
  68. # Print statistics if param -s/--statistics provided
  69. if flag_print_statistics:
  70. self.print_statistics()
  71. def get_file_information(self):
  72. """
  73. Returns a list of tuples, each containing a information of the file.
  74. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  75. """
  76. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  77. pdu_share = pdu_count / self.get_packet_count() * 100
  78. last_pdu_timestamp = self.process_db_query(
  79. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  80. return [("Pcap file path", self.pcap_filepath),
  81. ("Total packet count", self.get_packet_count(), "packets"),
  82. ("Recognized packets", self.get_packet_count() - pdu_count, "packets"),
  83. ("Unrecognized packets", pdu_count, "PDUs"),
  84. ("% Recognized packets", 100 - pdu_share, "%"),
  85. ("% Unrecognized packets", pdu_share, "%"),
  86. ("Last unknown PDU", last_pdu_timestamp),
  87. ("Capture duration", self.get_capture_duration(), "seconds"),
  88. ("Capture start", "\t" + str(self.get_pcap_timestamp_start())),
  89. ("Capture end", "\t" + str(self.get_pcap_timestamp_end()))]
  90. def get_general_file_statistics(self):
  91. """
  92. Returns a list of tuples, each containing a file statistic.
  93. :return: a list of tuples, each consisting of (description, value, unit).
  94. """
  95. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  96. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  97. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  98. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  99. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  100. @staticmethod
  101. def write_list(desc_val_unit_list, func, line_ending="\n"):
  102. """
  103. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three
  104. values and applies the function func on this string.
  105. Before generating the string, it identifies text containing a float number, casts the string to a
  106. float and rounds the value to two decimal digits.
  107. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  108. :param func: The function to be applied to each generated string
  109. :param line_ending: The formatting string to be applied at the end of each string
  110. """
  111. for entry in desc_val_unit_list:
  112. # Convert text containing float into float
  113. (description, value) = entry[0:2]
  114. if isinstance(value, str) and "." in value:
  115. try:
  116. value = float(value)
  117. except ValueError:
  118. pass # do nothing -> value was not a float
  119. # round float
  120. if isinstance(value, float):
  121. value = round(value, 4)
  122. # write into file
  123. if len(entry) == 3:
  124. unit = entry[2]
  125. func(description + ":\t" + str(value) + " " + unit + line_ending)
  126. else:
  127. func(description + ":\t" + str(value) + line_ending)
  128. def print_statistics(self):
  129. """
  130. Prints the basic file statistics to the terminal.
  131. """
  132. print("\nPCAP FILE INFORMATION ------------------------------")
  133. Statistics.write_list(self.get_file_information(), print, "")
  134. print("\nGENERAL FILE STATISTICS ----------------------------")
  135. Statistics.write_list(self.get_general_file_statistics(), print, "")
  136. print("\n")
  137. @staticmethod
  138. def calculate_entropy(frequency: list, normalized: bool = False):
  139. """
  140. Calculates entropy and normalized entropy of list of elements that have specific frequency
  141. :param frequency: The frequency of the elements.
  142. :param normalized: Calculate normalized entropy
  143. :return: entropy or (entropy, normalized entropy)
  144. """
  145. entropy, normalized_ent, n = 0, 0, 0
  146. sum_freq = sum(frequency)
  147. for i, x in enumerate(frequency):
  148. p_x = float(frequency[i] / sum_freq)
  149. if p_x > 0:
  150. n += 1
  151. entropy += - p_x * log(p_x, 2)
  152. if normalized:
  153. if log(n) > 0:
  154. normalized_ent = entropy / log(n, 2)
  155. return entropy, normalized_ent
  156. else:
  157. return entropy
  158. def calculate_complement_packet_rates(self, pps):
  159. """
  160. Calculates the complement packet rates of the background traffic packet rates for each interval.
  161. Then normalize it to maximum boundary, which is the input parameter pps
  162. :return: normalized packet rates for each time interval.
  163. """
  164. result = self.process_db_query(
  165. "SELECT lastPktTimestamp,pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  166. # print(result)
  167. bg_interval_pps = []
  168. complement_interval_pps = []
  169. intervals_sum = 0
  170. if result:
  171. # Get the interval in seconds
  172. for i, row in enumerate(result):
  173. if i < len(result) - 1:
  174. intervals_sum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
  175. interval = intervals_sum / (len(result) - 1)
  176. # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
  177. for row in result:
  178. bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
  179. # Find max PPS
  180. max_pps = max(bg_interval_pps, key=itemgetter(1))[1]
  181. for row in bg_interval_pps:
  182. complement_interval_pps.append((row[0], int(pps * (max_pps - row[1]) / max_pps)))
  183. return complement_interval_pps
  184. def get_tests_statistics(self):
  185. """
  186. Writes the calculated basic defects tests statistics into a file.
  187. """
  188. # self.stats_db.process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
  189. def count_frequncy(values_list):
  190. """
  191. TODO : FILL ME
  192. :param values_list:
  193. :return:
  194. """
  195. values, freq_output = [], []
  196. for x in values_list:
  197. if x in values:
  198. freq_output[values.index(x)] += 1
  199. else:
  200. values.append(x)
  201. freq_output.append(1)
  202. return values, freq_output
  203. # Payload Tests
  204. sum_payload_count = self.stats_db.process_user_defined_query("SELECT sum(payloadCount) FROM "
  205. "interval_statistics")
  206. pkt_count = self.stats_db.process_user_defined_query("SELECT packetCount FROM file_statistics")
  207. if sum_payload_count and pkt_count:
  208. payload_ratio = 0
  209. if pkt_count[0][0] != 0:
  210. payload_ratio = float(sum_payload_count[0][0] / pkt_count[0][0] * 100)
  211. else:
  212. payload_ratio = -1
  213. # TCP checksum Tests
  214. incorrect_checksum_count = self.stats_db.process_user_defined_query(
  215. "SELECT sum(incorrectTCPChecksumCount) FROM interval_statistics")
  216. correct_checksum_count = self.stats_db.process_user_defined_query(
  217. "SELECT avg(correctTCPChecksumCount) FROM interval_statistics")
  218. if incorrect_checksum_count and correct_checksum_count:
  219. incorrect_checksum_ratio = 0
  220. if (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) != 0:
  221. incorrect_checksum_ratio = float(incorrect_checksum_count[0][0] /
  222. (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) * 100)
  223. else:
  224. incorrect_checksum_ratio = -1
  225. # IP Src & Dst Tests
  226. result = self.stats_db.process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
  227. data, src_frequency, dst_frequency = [], [], []
  228. if result:
  229. for row in result:
  230. src_frequency.append(row[1])
  231. dst_frequency.append(row[2])
  232. ip_src_entropy, ip_src_norm_entropy = self.calculate_entropy(src_frequency, True)
  233. ip_dst_entropy, ip_dst_norm_entropy = self.calculate_entropy(dst_frequency, True)
  234. new_ip_count = self.stats_db.process_user_defined_query("SELECT newIPCount FROM interval_statistics")
  235. ip_novels_per_interval, ip_novels_per_interval_frequency = count_frequncy(new_ip_count)
  236. ip_novelty_dist_entropy = self.calculate_entropy(ip_novels_per_interval_frequency)
  237. # Ports Tests
  238. port0_count = self.stats_db.process_user_defined_query(
  239. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
  240. if not port0_count[0][0]:
  241. port0_count = 0
  242. else:
  243. port0_count = port0_count[0][0]
  244. # FIXME: could be extended
  245. reserved_port_count = self.stats_db.process_user_defined_query(
  246. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")
  247. if not reserved_port_count[0][0]:
  248. reserved_port_count = 0
  249. else:
  250. reserved_port_count = reserved_port_count[0][0]
  251. # TTL Tests
  252. result = self.stats_db.process_user_defined_query(
  253. "SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  254. data, frequency = [], []
  255. for row in result:
  256. frequency.append(row[1])
  257. ttl_entropy, ttl_norm_entropy = self.calculate_entropy(frequency, True)
  258. new_ttl_count = self.stats_db.process_user_defined_query("SELECT newTTLCount FROM interval_statistics")
  259. ttl_novels_per_interval, ttl_novels_per_interval_frequency = count_frequncy(new_ttl_count)
  260. ttl_novelty_dist_entropy = self.calculate_entropy(ttl_novels_per_interval_frequency)
  261. # Window Size Tests
  262. result = self.stats_db.process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
  263. data, frequency = [], []
  264. for row in result:
  265. frequency.append(row[1])
  266. win_entropy, win_norm_entropy = self.calculate_entropy(frequency, True)
  267. new_win_size_count = self.stats_db.process_user_defined_query("SELECT newWinSizeCount FROM interval_statistics")
  268. win_novels_per_interval, win_novels_per_interval_frequency = count_frequncy(new_win_size_count)
  269. win_novelty_dist_entropy = self.calculate_entropy(win_novels_per_interval_frequency)
  270. # ToS Tests
  271. result = self.stats_db.process_user_defined_query(
  272. "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
  273. data, frequency = [], []
  274. for row in result:
  275. frequency.append(row[1])
  276. tos_entropy, tos_norm_entropy = self.calculate_entropy(frequency, True)
  277. new_tos_count = self.stats_db.process_user_defined_query("SELECT newToSCount FROM interval_statistics")
  278. tos_novels_per_interval, tos_novels_per_interval_frequency = count_frequncy(new_tos_count)
  279. tos_novelty_dist_entropy = self.calculate_entropy(tos_novels_per_interval_frequency)
  280. # MSS Tests
  281. result = self.stats_db.process_user_defined_query(
  282. "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  283. data, frequency = [], []
  284. for row in result:
  285. frequency.append(row[1])
  286. mss_entropy, mss_norm_entropy = self.calculate_entropy(frequency, True)
  287. new_mss_count = self.stats_db.process_user_defined_query("SELECT newMSSCount FROM interval_statistics")
  288. mss_novels_per_interval, mss_novels_per_interval_frequency = count_frequncy(new_mss_count)
  289. mss_novelty_dist_entropy = self.calculate_entropy(mss_novels_per_interval_frequency)
  290. result = self.stats_db.process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
  291. # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
  292. if not result[0][0]:
  293. result = 0
  294. else:
  295. result = result[0][0]
  296. big_mss = (result / sum(frequency)) * 100
  297. output = []
  298. if self.do_extra_tests:
  299. output = [("Payload ratio", payload_ratio, "%"),
  300. ("Incorrect TCP checksum ratio", incorrect_checksum_ratio, "%")]
  301. output = output + [("# IP addresses", sum([x[0] for x in new_ip_count]), ""),
  302. ("IP Src Entropy", ip_src_entropy, ""),
  303. ("IP Src Normalized Entropy", ip_src_norm_entropy, ""),
  304. ("IP Dst Entropy", ip_dst_entropy, ""),
  305. ("IP Dst Normalized Entropy", ip_dst_norm_entropy, ""),
  306. ("IP Novelty Distribution Entropy", ip_novelty_dist_entropy, ""),
  307. ("# TTL values", sum([x[0] for x in new_ttl_count]), ""),
  308. ("TTL Entropy", ttl_entropy, ""),
  309. ("TTL Normalized Entropy", ttl_norm_entropy, ""),
  310. ("TTL Novelty Distribution Entropy", ttl_novelty_dist_entropy, ""),
  311. ("# WinSize values", sum([x[0] for x in new_win_size_count]), ""),
  312. ("WinSize Entropy", win_entropy, ""),
  313. ("WinSize Normalized Entropy", win_norm_entropy, ""),
  314. ("WinSize Novelty Distribution Entropy", win_novelty_dist_entropy, ""),
  315. ("# ToS values", sum([x[0] for x in new_tos_count]), ""),
  316. ("ToS Entropy", tos_entropy, ""),
  317. ("ToS Normalized Entropy", tos_norm_entropy, ""),
  318. ("ToS Novelty Distribution Entropy", tos_novelty_dist_entropy, ""),
  319. ("# MSS values", sum([x[0] for x in new_mss_count]), ""),
  320. ("MSS Entropy", mss_entropy, ""),
  321. ("MSS Normalized Entropy", mss_norm_entropy, ""),
  322. ("MSS Novelty Distribution Entropy", mss_novelty_dist_entropy, ""),
  323. ("======================", "", "")]
  324. # Reasoning the statistics values
  325. if self.do_extra_tests:
  326. if payload_ratio > 80:
  327. output.append(("WARNING: Too high payload ratio", payload_ratio, "%."))
  328. if payload_ratio < 30:
  329. output.append(("WARNING: Too low payload ratio", payload_ratio, "% (Injecting attacks that are carried "
  330. "out in the packet payloads is not "
  331. "recommmanded)."))
  332. if incorrect_checksum_ratio > 5:
  333. output.append(("WARNING: High incorrect TCP checksum ratio", incorrect_checksum_ratio, "%."))
  334. if ip_src_norm_entropy > 0.65:
  335. output.append(("WARNING: High IP source normalized entropy", ip_src_norm_entropy, "."))
  336. if ip_src_norm_entropy < 0.2:
  337. output.append(("WARNING: Low IP source normalized entropy", ip_src_norm_entropy, "."))
  338. if ip_dst_norm_entropy > 0.65:
  339. output.append(("WARNING: High IP destination normalized entropy", ip_dst_norm_entropy, "."))
  340. if ip_dst_norm_entropy < 0.2:
  341. output.append(("WARNING: Low IP destination normalized entropy", ip_dst_norm_entropy, "."))
  342. if ttl_norm_entropy > 0.65:
  343. output.append(("WARNING: High TTL normalized entropy", ttl_norm_entropy, "."))
  344. if ttl_norm_entropy < 0.2:
  345. output.append(("WARNING: Low TTL normalized entropy", ttl_norm_entropy, "."))
  346. if ttl_novelty_dist_entropy < 1:
  347. output.append(("WARNING: Too low TTL novelty distribution entropy", ttl_novelty_dist_entropy,
  348. "(The distribution of the novel TTL values is suspicious)."))
  349. if win_norm_entropy > 0.6:
  350. output.append(("WARNING: High Window Size normalized entropy", win_norm_entropy, "."))
  351. if win_norm_entropy < 0.1:
  352. output.append(("WARNING: Low Window Size normalized entropy", win_norm_entropy, "."))
  353. if win_novelty_dist_entropy < 4:
  354. output.append(("WARNING: Low Window Size novelty distribution entropy", win_novelty_dist_entropy,
  355. "(The distribution of the novel Window Size values is suspicious)."))
  356. if tos_norm_entropy > 0.4:
  357. output.append(("WARNING: High ToS normalized entropy", tos_norm_entropy, "."))
  358. if tos_norm_entropy < 0.1:
  359. output.append(("WARNING: Low ToS normalized entropy", tos_norm_entropy, "."))
  360. if tos_novelty_dist_entropy < 0.5:
  361. output.append(("WARNING: Low ToS novelty distribution entropy", tos_novelty_dist_entropy,
  362. "(The distribution of the novel ToS values is suspicious)."))
  363. if mss_norm_entropy > 0.4:
  364. output.append(("WARNING: High MSS normalized entropy", mss_norm_entropy, "."))
  365. if mss_norm_entropy < 0.1:
  366. output.append(("WARNING: Low MSS normalized entropy", mss_norm_entropy, "."))
  367. if mss_novelty_dist_entropy < 0.5:
  368. output.append(("WARNING: Low MSS novelty distribution entropy", mss_novelty_dist_entropy,
  369. "(The distribution of the novel MSS values is suspicious)."))
  370. if big_mss > 50:
  371. output.append(("WARNING: High ratio of MSS > 1460", big_mss, "% (High fragmentation rate in Ethernet)."))
  372. if port0_count > 0:
  373. output.append(("WARNING: Port number 0 is used in ", port0_count, "packets (awkward-looking port)."))
  374. if reserved_port_count > 0:
  375. output.append(("WARNING: Reserved port numbers are used in ", reserved_port_count,
  376. "packets (uncommonly-used ports)."))
  377. return output
  378. def write_statistics_to_file(self):
  379. """
  380. Writes the calculated basic statistics into a file.
  381. """
  382. def _write_header(title: str):
  383. """
  384. Writes the section header into the open file.
  385. :param title: The section title
  386. """
  387. target.write("====================== \n")
  388. target.write(title + " \n")
  389. target.write("====================== \n")
  390. target = open(self.pcap_filepath + ".stat", 'w')
  391. target.truncate()
  392. _write_header("PCAP file information")
  393. Statistics.write_list(self.get_file_information(), target.write)
  394. _write_header("General statistics")
  395. Statistics.write_list(self.get_general_file_statistics(), target.write)
  396. _write_header("Tests statistics")
  397. Statistics.write_list(self.get_tests_statistics(), target.write)
  398. target.close()
  399. def get_capture_duration(self):
  400. """
  401. :return: The duration of the capture in seconds
  402. """
  403. return self.file_info['captureDuration']
  404. def get_pcap_timestamp_start(self):
  405. """
  406. :return: The timestamp of the first packet in the PCAP file
  407. """
  408. return self.file_info['timestampFirstPacket']
  409. def get_pcap_timestamp_end(self):
  410. """
  411. :return: The timestamp of the last packet in the PCAP file
  412. """
  413. return self.file_info['timestampLastPacket']
  414. def get_pps_sent(self, ip_address: str):
  415. """
  416. Calculates the sent packets per seconds for a given IP address.
  417. :param ip_address: The IP address whose packets per second should be calculated
  418. :return: The sent packets per seconds for the given IP address
  419. """
  420. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  421. (ip_address,))
  422. capture_duration = float(self.get_capture_duration())
  423. return int(float(packets_sent) / capture_duration)
  424. def get_pps_received(self, ip_address: str):
  425. """
  426. Calculate the packets per second received for a given IP address.
  427. :param ip_address: The IP address used for the calculation
  428. :return: The number of packets per second received
  429. """
  430. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  431. False,
  432. (ip_address,))
  433. capture_duration = float(self.get_capture_duration())
  434. return int(float(packets_received) / capture_duration)
  435. def get_packet_count(self):
  436. """
  437. :return: The number of packets in the loaded PCAP file
  438. """
  439. return self.file_info['packetCount']
  440. def get_most_used_ip_address(self):
  441. """
  442. :return: The IP address/addresses with the highest sum of packets sent and received
  443. """
  444. return Util.handle_most_used_outputs(self.process_db_query("most_used(ipAddress)"))
  445. def get_ttl_distribution(self, ip_address: str):
  446. """
  447. TODO: FILL ME
  448. :param ip_address:
  449. :return:
  450. """
  451. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ip_address + '"')
  452. result_dict = {key: value for (key, value) in result}
  453. return result_dict
  454. def get_mss_distribution(self, ip_address: str):
  455. """
  456. TODO: FILL ME
  457. :param ip_address:
  458. :return:
  459. """
  460. result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ip_address + '"')
  461. result_dict = {key: value for (key, value) in result}
  462. return result_dict
  463. def get_win_distribution(self, ip_address: str):
  464. """
  465. TODO: FILL ME
  466. :param ip_address:
  467. :return:
  468. """
  469. result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ip_address + '"')
  470. result_dict = {key: value for (key, value) in result}
  471. return result_dict
  472. def get_tos_distribution(self, ip_address: str):
  473. """
  474. TODO: FILL ME
  475. :param ip_address:
  476. :return:
  477. """
  478. result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ip_address + '"')
  479. result_dict = {key: value for (key, value) in result}
  480. return result_dict
  481. def get_ip_address_count(self):
  482. """
  483. TODO: FILL ME
  484. :return:
  485. """
  486. return self.process_db_query("SELECT COUNT(*) FROM ip_statistics")
  487. def get_ip_addresses(self):
  488. """
  489. TODO: FILL ME
  490. :return:
  491. """
  492. return self.process_db_query("SELECT ipAddress FROM ip_statistics")
  493. def get_random_ip_address(self, count: int = 1):
  494. """
  495. :param count: The number of IP addresses to return
  496. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of
  497. randomly chosen IP addresses
  498. """
  499. ip_address_list = self.process_db_query("SELECT ipAddress from ip_statistics ORDER BY ipAddress ASC")
  500. if count == 1:
  501. return random.choice(ip_address_list)
  502. else:
  503. result_list = []
  504. for i in range(0, count):
  505. random_ip = random.choice(ip_address_list)
  506. result_list.append(random_ip)
  507. ip_address_list.remove(random_ip)
  508. return result_list
  509. def get_ip_address_from_mac(self, mac_address: str):
  510. """
  511. :param mac_address: the MAC address of which the IP shall be returned, if existing in DB
  512. :return: the IP address used in the dataset by a given MAC address
  513. """
  514. return self.process_db_query("SELECT DISTINCT ipAddress FROM ip_mac WHERE macAddress = '" + mac_address + "'")
  515. def get_mac_address(self, ip_address: str):
  516. """
  517. :return: The MAC address used in the dataset for the given IP address.
  518. """
  519. return self.process_db_query("SELECT DISTINCT macAddress from ip_mac WHERE ipAddress = '" + ip_address + "'")
  520. def get_most_used_ttl_value(self):
  521. """
  522. :return: The most used TTL value.
  523. """
  524. return self.process_db_query("SELECT ttlValue FROM (SELECT ttlValue, SUM(ttlCount) as occ FROM ip_ttl GROUP BY "
  525. "ttlValue) WHERE occ=(SELECT SUM(ttlCount) as occ FROM ip_ttl GROUP BY ttlValue "
  526. "ORDER BY occ DESC LIMIT 1) ORDER BY ttlValue ASC")
  527. def get_most_used_ip_class(self):
  528. """
  529. :return: The most used IP class.
  530. """
  531. return self.process_db_query("SELECT ipClass FROM (SELECT ipClass, COUNT(*) as occ from ip_statistics GROUP BY "
  532. "ipClass ORDER BY occ DESC) WHERE occ=(SELECT COUNT(*) as occ from ip_statistics "
  533. "GROUP BY ipClass ORDER BY occ DESC LIMIT 1) ORDER BY ipClass ASC")
  534. def get_most_used_win_size(self):
  535. """
  536. :return: The most used window size.
  537. """
  538. return self.process_db_query("SELECT winSize FROM (SELECT winSize, SUM(winCount) as occ FROM tcp_win GROUP BY "
  539. "winSize) WHERE occ=(SELECT SUM(winCount) as occ FROM tcp_win GROUP BY winSize "
  540. "ORDER BY occ DESC LIMIT 1) ORDER BY winSize ASC")
  541. def get_most_used_mss_value(self):
  542. """
  543. :return: The most used mss value.
  544. """
  545. return self.process_db_query("SELECT mssValue FROM (SELECT mssValue, SUM(mssCount) as occ FROM tcp_mss GROUP BY"
  546. " mssValue) WHERE occ=(SELECT SUM(mssCount) as occ FROM tcp_mss GROUP BY mssValue "
  547. "ORDER BY occ DESC LIMIT 1) ORDER BY mssValue ASC")
  548. def get_most_used_mss(self, ip_address: str):
  549. """
  550. :param ip_address: The IP address whose used MSS should be determined
  551. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  552. then None is returned
  553. """
  554. mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ip_address +
  555. '" AND mssCount == (SELECT MAX(mssCount) from tcp_mss WHERE ipAddress="'
  556. + ip_address + '")')
  557. if isinstance(mss_value, int):
  558. return mss_value
  559. elif isinstance(mss_value, list):
  560. if len(mss_value) == 0:
  561. return None
  562. else:
  563. mss_value.sort()
  564. return mss_value[0]
  565. else:
  566. return None
  567. def get_most_used_ttl(self, ip_address: str):
  568. """
  569. :param ip_address: The IP address whose used TTL should be determined
  570. :return: The TTL value used by the IP address, or if the IP addresses never specified a TTL,
  571. then None is returned
  572. """
  573. ttl_value = self.process_db_query('SELECT ttlValue from ip_ttl WHERE ipAddress="' + ip_address +
  574. '" AND ttlCount == (SELECT MAX(ttlCount) from ip_ttl WHERE ipAddress="'
  575. + ip_address + '")')
  576. if isinstance(ttl_value, int):
  577. return ttl_value
  578. elif isinstance(ttl_value, list):
  579. if len(ttl_value) == 0:
  580. return None
  581. else:
  582. ttl_value.sort()
  583. return ttl_value[0]
  584. else:
  585. return None
  586. def get_rnd_win_size(self, pkts_num):
  587. """
  588. :param pkts_num: maximum number of window sizes, that should be returned
  589. :return: A list of randomly chosen window sizes with given length.
  590. """
  591. sql_return = self.process_db_query("SELECT DISTINCT winSize FROM tcp_win ORDER BY winsize ASC;")
  592. if not isinstance(sql_return, list):
  593. return [sql_return]
  594. result = []
  595. for i in range(0, min(pkts_num, len(sql_return))):
  596. result.append(random.choice(sql_return))
  597. sql_return.remove(result[i])
  598. return result
  599. def get_statistics_database(self):
  600. """
  601. :return: A reference to the statistics database object
  602. """
  603. return self.stats_db
  604. def process_db_query(self, query_string_in: str, print_results: bool = False):
  605. """
  606. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  607. query.
  608. :param query_string_in: The query to be processed
  609. :param print_results: Indicates whether the results should be printed to terminal
  610. :return: The result of the query
  611. """
  612. return self.stats_db.process_db_query(query_string_in, print_results)
  613. def is_query(self, value: str):
  614. """
  615. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  616. :param value: The string to be checked
  617. :return: True if the string is recognized as a query, otherwise False.
  618. """
  619. if not isinstance(value, str):
  620. return False
  621. else:
  622. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  623. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  624. @staticmethod
  625. def calculate_standard_deviation(lst):
  626. """
  627. Calculates the standard deviation of a list of numbers.
  628. :param lst: The list of numbers to calculate its SD.
  629. """
  630. num_items = len(lst)
  631. mean = sum(lst) / num_items
  632. differences = [x - mean for x in lst]
  633. sq_differences = [d ** 2 for d in differences]
  634. ssd = sum(sq_differences)
  635. variance = ssd / num_items
  636. sd = sqrt(variance)
  637. return sd
  638. def plot_statistics(self, entropy: int, file_format: str = 'pdf'): # 'png'
  639. """
  640. Plots the statistics associated with the dataset.
  641. :param entropy: the statistics entropy
  642. :param file_format: The format to be used to save the statistics diagrams.
  643. """
  644. def plot_distribution(query_output, title, x_label, y_label, file_ending: str):
  645. """
  646. TODO: FILL ME
  647. :param query_output:
  648. :param title:
  649. :param x_label:
  650. :param y_label:
  651. :param file_ending:
  652. :return:
  653. """
  654. plt.gcf().clear()
  655. graphx, graphy = [], []
  656. for row in query_output:
  657. graphx.append(row[0])
  658. graphy.append(row[1])
  659. plt.autoscale(enable=True, axis='both')
  660. plt.title(title)
  661. plt.xlabel(x_label)
  662. plt.ylabel(y_label)
  663. width = 0.1
  664. plt.xlim([0, (max(graphx) * 1.1)])
  665. plt.grid(True)
  666. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  667. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  668. plt.savefig(out, dpi=500)
  669. return out
  670. def plot_ttl(file_ending: str):
  671. """
  672. TODO: FILL ME
  673. :param file_ending:
  674. :return:
  675. """
  676. query_output = self.stats_db.process_user_defined_query(
  677. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  678. title = "TTL Distribution"
  679. x_label = "TTL Value"
  680. y_label = "Number of Packets"
  681. if query_output:
  682. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  683. def plot_mss(file_ending: str):
  684. """
  685. TODO: FILL ME
  686. :param file_ending:
  687. :return:
  688. """
  689. query_output = self.stats_db.process_user_defined_query(
  690. "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  691. title = "MSS Distribution"
  692. x_label = "MSS Value"
  693. y_label = "Number of Packets"
  694. if query_output:
  695. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  696. def plot_win(file_ending: str):
  697. """
  698. TODO: FILL ME
  699. :param file_ending:
  700. :return:
  701. """
  702. query_output = self.stats_db.process_user_defined_query(
  703. "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
  704. title = "Window Size Distribution"
  705. x_label = "Window Size"
  706. y_label = "Number of Packets"
  707. if query_output:
  708. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  709. def plot_protocol(file_ending: str):
  710. """
  711. TODO: FILL ME
  712. :param file_ending:
  713. :return:
  714. """
  715. plt.gcf().clear()
  716. result = self.stats_db.process_user_defined_query(
  717. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  718. if result:
  719. graphx, graphy = [], []
  720. for row in result:
  721. graphx.append(row[0])
  722. graphy.append(row[1])
  723. plt.autoscale(enable=True, axis='both')
  724. plt.title("Protocols Distribution")
  725. plt.xlabel('Protocols')
  726. plt.ylabel('Number of Packets')
  727. width = 0.1
  728. plt.xlim([0, len(graphx)])
  729. plt.grid(True)
  730. # Protocols' names on x-axis
  731. x = range(0, len(graphx))
  732. my_xticks = graphx
  733. plt.xticks(x, my_xticks)
  734. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  735. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  736. plt.savefig(out, dpi=500)
  737. return out
  738. else:
  739. print("Error plot protocol: No protocol values found!")
  740. def plot_port(file_ending: str):
  741. """
  742. TODO: FILL ME
  743. :param file_ending:
  744. :return:
  745. """
  746. plt.gcf().clear()
  747. result = self.stats_db.process_user_defined_query(
  748. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  749. graphx, graphy = [], []
  750. for row in result:
  751. graphx.append(row[0])
  752. graphy.append(row[1])
  753. plt.autoscale(enable=True, axis='both')
  754. plt.title("Ports Distribution")
  755. plt.xlabel('Ports Numbers')
  756. plt.ylabel('Number of Packets')
  757. width = 0.1
  758. plt.xlim([0, max(graphx)])
  759. plt.grid(True)
  760. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  761. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  762. plt.savefig(out, dpi=500)
  763. return out
  764. # This distribution is not drawable for big datasets
  765. def plot_ip_src(file_ending: str):
  766. """
  767. TODO: FILL ME
  768. :param file_ending:
  769. :return:
  770. """
  771. plt.gcf().clear()
  772. result = self.stats_db.process_user_defined_query(
  773. "SELECT ipAddress, pktsSent FROM ip_statistics")
  774. graphx, graphy = [], []
  775. for row in result:
  776. graphx.append(row[0])
  777. graphy.append(row[1])
  778. plt.autoscale(enable=True, axis='both')
  779. plt.title("Source IP Distribution")
  780. plt.xlabel('Source IP')
  781. plt.ylabel('Number of Packets')
  782. width = 0.1
  783. plt.xlim([0, len(graphx)])
  784. plt.grid(True)
  785. # IPs on x-axis
  786. x = range(0, len(graphx))
  787. my_xticks = graphx
  788. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  789. plt.tight_layout()
  790. # limit the number of xticks
  791. plt.locator_params(axis='x', nbins=20)
  792. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  793. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  794. plt.savefig(out, dpi=500)
  795. return out
  796. # This distribution is not drawable for big datasets
  797. def plot_ip_dst(file_ending: str):
  798. """
  799. TODO: FILL ME
  800. :param file_ending:
  801. :return:
  802. """
  803. plt.gcf().clear()
  804. result = self.stats_db.process_user_defined_query(
  805. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  806. graphx, graphy = [], []
  807. for row in result:
  808. graphx.append(row[0])
  809. graphy.append(row[1])
  810. plt.autoscale(enable=True, axis='both')
  811. plt.title("Destination IP Distribution")
  812. plt.xlabel('Destination IP')
  813. plt.ylabel('Number of Packets')
  814. width = 0.1
  815. plt.xlim([0, len(graphx)])
  816. plt.grid(True)
  817. # IPs on x-axis
  818. x = range(0, len(graphx))
  819. my_xticks = graphx
  820. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  821. plt.tight_layout()
  822. # limit the number of xticks
  823. plt.locator_params(axis='x', nbins=20)
  824. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  825. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  826. plt.savefig(out, dpi=500)
  827. return out
  828. def plot_interval_statistics(query_output, title, x_label, y_label, file_ending: str):
  829. """
  830. TODO: FILL ME
  831. :param query_output:
  832. :param title:
  833. :param x_label:
  834. :param y_label:
  835. :param file_ending:
  836. :return:
  837. """
  838. plt.gcf().clear()
  839. graphx, graphy = [], []
  840. for row in query_output:
  841. graphx.append(row[0])
  842. graphy.append(row[1])
  843. plt.autoscale(enable=True, axis='both')
  844. plt.title(title)
  845. plt.xlabel(x_label)
  846. plt.ylabel(y_label)
  847. width = 0.5
  848. plt.xlim([0, len(graphx)])
  849. plt.grid(True)
  850. # timestamp on x-axis
  851. x = range(0, len(graphx))
  852. # limit the number of xticks
  853. plt.locator_params(axis='x', nbins=20)
  854. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  855. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  856. plt.savefig(out, dpi=500)
  857. return out
  858. def plot_interval_pkt_count(file_ending: str):
  859. """
  860. TODO: FILL ME
  861. :param file_ending:
  862. :return:
  863. """
  864. query_output = self.stats_db.process_user_defined_query(
  865. "SELECT lastPktTimestamp, pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  866. title = "Packet Rate"
  867. x_label = "Time Interval"
  868. y_label = "Number of Packets"
  869. if query_output:
  870. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  871. def plot_interval_ip_src_ent(file_ending: str):
  872. """
  873. TODO: FILL ME
  874. :param file_ending:
  875. :return:
  876. """
  877. query_output = self.stats_db.process_user_defined_query(
  878. "SELECT lastPktTimestamp, ipSrcEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  879. title = "Source IP Entropy"
  880. x_label = "Time Interval"
  881. y_label = "Entropy"
  882. if query_output:
  883. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  884. def plot_interval_ip_dst_ent(file_ending: str):
  885. """
  886. TODO: FILL ME
  887. :param file_ending:
  888. :return:
  889. """
  890. query_output = self.stats_db.process_user_defined_query(
  891. "SELECT lastPktTimestamp, ipDstEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  892. title = "Destination IP Entropy"
  893. x_label = "Time Interval"
  894. y_label = "Entropy"
  895. if query_output:
  896. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  897. def plot_interval_new_ip(file_ending: str):
  898. """
  899. TODO: FILL ME
  900. :param file_ending:
  901. :return:
  902. """
  903. query_output = self.stats_db.process_user_defined_query(
  904. "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
  905. title = "IP Novelty Distribution"
  906. x_label = "Time Interval"
  907. y_label = "Novel values count"
  908. if query_output:
  909. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  910. def plot_interval_new_port(file_ending: str):
  911. """
  912. TODO: FILL ME
  913. :param file_ending:
  914. :return:
  915. """
  916. query_output = self.stats_db.process_user_defined_query(
  917. "SELECT lastPktTimestamp, newPortCount FROM interval_statistics ORDER BY lastPktTimestamp")
  918. title = "Port Novelty Distribution"
  919. x_label = "Time Interval"
  920. y_label = "Novel values count"
  921. if query_output:
  922. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  923. def plot_interval_new_ttl(file_ending: str):
  924. """
  925. TODO: FILL ME
  926. :param file_ending:
  927. :return:
  928. """
  929. query_output = self.stats_db.process_user_defined_query(
  930. "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
  931. title = "TTL Novelty Distribution"
  932. x_label = "Time Interval"
  933. y_label = "Novel values count"
  934. if query_output:
  935. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  936. def plot_interval_new_tos(file_ending: str):
  937. """
  938. TODO: FILL ME
  939. :param file_ending:
  940. :return:
  941. """
  942. query_output = self.stats_db.process_user_defined_query(
  943. "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  944. title = "ToS Novelty Distribution"
  945. x_label = "Time Interval"
  946. y_label = "Novel values count"
  947. if query_output:
  948. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  949. def plot_interval_new_win_size(file_ending: str):
  950. """
  951. TODO: FILL ME
  952. :param file_ending:
  953. :return:
  954. """
  955. query_output = self.stats_db.process_user_defined_query(
  956. "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
  957. title = "Window Size Novelty Distribution"
  958. x_label = "Time Interval"
  959. y_label = "Novel values count"
  960. if query_output:
  961. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  962. def plot_interval_new_mss(file_ending: str):
  963. """
  964. TODO: FILL ME
  965. :param file_ending:
  966. :return:
  967. """
  968. query_output = self.stats_db.process_user_defined_query(
  969. "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  970. title = "MSS Novelty Distribution"
  971. x_label = "Time Interval"
  972. y_label = "Novel values count"
  973. if query_output:
  974. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  975. def plot_interval_ip_dst_cum_ent(file_ending: str):
  976. """
  977. TODO: FILL ME
  978. :param file_ending:
  979. :return:
  980. """
  981. plt.gcf().clear()
  982. result = self.stats_db.process_user_defined_query(
  983. "SELECT lastPktTimestamp, ipDstCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  984. graphx, graphy = [], []
  985. for row in result:
  986. graphx.append(row[0])
  987. graphy.append(row[1])
  988. # If entropy was not calculated do not plot the graph
  989. if graphy[0] != -1:
  990. plt.autoscale(enable=True, axis='both')
  991. plt.title("Destination IP Cumulative Entropy")
  992. # plt.xlabel('Timestamp')
  993. plt.xlabel('Time Interval')
  994. plt.ylabel('Entropy')
  995. plt.xlim([0, len(graphx)])
  996. plt.grid(True)
  997. # timestamp on x-axis
  998. x = range(0, len(graphx))
  999. # my_xticks = graphx
  1000. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  1001. # plt.tight_layout()
  1002. # limit the number of xticks
  1003. plt.locator_params(axis='x', nbins=20)
  1004. plt.plot(x, graphy, 'r')
  1005. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-cum-ent' + file_ending)
  1006. plt.savefig(out, dpi=500)
  1007. return out
  1008. def plot_interval_ip_src_cum_ent(file_ending: str):
  1009. """
  1010. TODO: FILL ME
  1011. :param file_ending:
  1012. :return:
  1013. """
  1014. plt.gcf().clear()
  1015. result = self.stats_db.process_user_defined_query(
  1016. "SELECT lastPktTimestamp, ipSrcCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  1017. graphx, graphy = [], []
  1018. for row in result:
  1019. graphx.append(row[0])
  1020. graphy.append(row[1])
  1021. # If entropy was not calculated do not plot the graph
  1022. if graphy[0] != -1:
  1023. plt.autoscale(enable=True, axis='both')
  1024. plt.title("Source IP Cumulative Entropy")
  1025. # plt.xlabel('Timestamp')
  1026. plt.xlabel('Time Interval')
  1027. plt.ylabel('Entropy')
  1028. plt.xlim([0, len(graphx)])
  1029. plt.grid(True)
  1030. # timestamp on x-axis
  1031. x = range(0, len(graphx))
  1032. # my_xticks = graphx
  1033. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  1034. # plt.tight_layout()
  1035. # limit the number of xticks
  1036. plt.locator_params(axis='x', nbins=20)
  1037. plt.plot(x, graphy, 'r')
  1038. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-cum-ent' + file_ending)
  1039. plt.savefig(out, dpi=500)
  1040. return out
  1041. ttl_out_path = plot_ttl('.' + file_format)
  1042. print(".", end="", flush=True)
  1043. mss_out_path = plot_mss('.' + file_format)
  1044. print(".", end="", flush=True)
  1045. win_out_path = plot_win('.' + file_format)
  1046. print(".", end="", flush=True)
  1047. protocol_out_path = plot_protocol('.' + file_format)
  1048. print(".", end="", flush=True)
  1049. plot_interval_pktCount = plot_interval_pkt_count('.' + file_format)
  1050. print(".", end="", flush=True)
  1051. if entropy:
  1052. plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + file_format)
  1053. print(".", end="", flush=True)
  1054. plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + file_format)
  1055. print(".", end="", flush=True)
  1056. plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + file_format)
  1057. print(".", end="", flush=True)
  1058. plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + file_format)
  1059. print(".", end="", flush=True)
  1060. plot_interval_new_ip = plot_interval_new_ip('.' + file_format)
  1061. print(".", end="", flush=True)
  1062. plot_interval_new_port = plot_interval_new_port('.' + file_format)
  1063. print(".", end="", flush=True)
  1064. plot_interval_new_ttl = plot_interval_new_ttl('.' + file_format)
  1065. print(".", end="", flush=True)
  1066. plot_interval_new_tos = plot_interval_new_tos('.' + file_format)
  1067. print(".", end="", flush=True)
  1068. plot_interval_new_win_size = plot_interval_new_win_size('.' + file_format)
  1069. print(".", end="", flush=True)
  1070. plot_interval_new_mss = plot_interval_new_mss('.' + file_format)
  1071. print(" done.")
  1072. # Time consuming plot
  1073. # port_out_path = plot_port('.' + format)
  1074. # Not drawable for too many IPs
  1075. # ip_src_out_path = plot_ip_src('.' + format)
  1076. # ip_dst_out_path = plot_ip_dst('.' + format)
  1077. print("Saved plots in the input PCAP directory.")
  1078. def stats_summary_post_attack(self, added_packets):
  1079. """
  1080. Prints a summary of relevant statistics after an attack is injected
  1081. :param added_packets: sum of packets added by attacks, gets updated if more than one attack
  1082. :return: None
  1083. """
  1084. total_packet_count = self.get_packet_count() + added_packets
  1085. added_packets_share = added_packets / total_packet_count * 100
  1086. timespan = self.get_capture_duration()
  1087. summary = [("Total packet count", total_packet_count, "packets"),
  1088. ("Added packet count", added_packets, "packets"),
  1089. ("Share of added packets", added_packets_share, "%"),
  1090. ("Capture duration", timespan, "seconds")]
  1091. print("\nPOST INJECTION STATISTICS SUMMARY --------------------------")
  1092. self.write_list(summary, print, "")
  1093. print("------------------------------------------------------------")
  1094. def stats_summary_new_db(self):
  1095. """
  1096. Prints a summary of relevant statistics when a new db is created
  1097. :return: None
  1098. """
  1099. self.file_info = self.stats_db.get_file_info()
  1100. print("\nNew database has been generated, printing statistics summary... ")
  1101. total_packet_count = self.get_packet_count()
  1102. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  1103. if pdu_count is None:
  1104. pdu_count = 0
  1105. pdu_share = pdu_count / total_packet_count * 100
  1106. last_pdu_timestamp = self.process_db_query(
  1107. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  1108. timespan = self.get_capture_duration()
  1109. summary = [("Total packet count", total_packet_count, "packets"),
  1110. ("Recognized packets", total_packet_count - pdu_count, "packets"),
  1111. ("Unrecognized packets", pdu_count, "PDUs"),
  1112. ("% Recognized packets", 100 - pdu_share, "%"),
  1113. ("% Unrecognized packets", pdu_share, "%"),
  1114. ("Last unknown PDU", last_pdu_timestamp),
  1115. ("Capture duration", timespan, "seconds")]
  1116. print("\nPCAP FILE STATISTICS SUMMARY ------------------------------")
  1117. self.write_list(summary, print, "")
  1118. print("------------------------------------------------------------")