Statistics.py 80 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836
  1. import os
  2. import random
  3. import time
  4. import numpy
  5. from math import sqrt, ceil, log
  6. from operator import itemgetter
  7. # TODO: double check this import
  8. # does it complain because libpcapreader is not a .py?
  9. import ID2TLib.libpcapreader as pr
  10. import Core.StatsDatabase as statsDB
  11. import ID2TLib.PcapFile as PcapFile
  12. import ID2TLib.Utility as Util
  13. from ID2TLib.IPv4 import IPAddress
  14. import matplotlib.pyplot as plt
  15. class Statistics:
  16. def __init__(self, pcap_file: PcapFile.PcapFile):
  17. """
  18. Creates a new Statistics object.
  19. :param pcap_file: A reference to the PcapFile object
  20. """
  21. # Fields
  22. self.pcap_filepath = pcap_file.pcap_file_path
  23. self.pcap_proc = None
  24. self.do_extra_tests = False
  25. self.file_info = None
  26. # Create folder for statistics database if required
  27. self.path_db = pcap_file.get_db_path()
  28. path_dir = os.path.dirname(self.path_db)
  29. if not os.path.isdir(path_dir):
  30. os.makedirs(path_dir)
  31. # Class instances
  32. self.stats_db = statsDB.StatsDatabase(self.path_db)
  33. def list_previous_interval_statistic_tables(self, output: bool=True):
  34. """
  35. Prints a list of all interval statistic tables from the database.
  36. :return: A list of intervals in seconds used to create the previous interval statistics tables
  37. """
  38. if self.stats_db.process_db_query("SELECT name FROM sqlite_master WHERE name='interval_tables';"):
  39. previous_interval_tables = self.stats_db.process_db_query("SELECT * FROM interval_tables;")
  40. else:
  41. previous_interval_tables = self.stats_db.process_db_query("SELECT name FROM sqlite_master WHERE "
  42. "type='table' AND name LIKE "
  43. "'interval_statistics_%';")
  44. previous_intervals = []
  45. if previous_interval_tables:
  46. if not isinstance(previous_interval_tables, list):
  47. previous_interval_tables = [previous_interval_tables]
  48. if output:
  49. print("There are " + str(len(previous_interval_tables)) + " interval statistics table(s) in the "
  50. "database:")
  51. i = 0
  52. if output:
  53. print("ID".ljust(3) + " | " + "interval in seconds".ljust(30) + " | is_default", end="")
  54. if len(previous_interval_tables) > 0 and len(previous_interval_tables[0]) > 2:
  55. print( " | extra_tests")
  56. else:
  57. print("")
  58. for table in previous_interval_tables:
  59. seconds = float(table[0][len("interval_statistics_"):])/1000000
  60. if output:
  61. print(str(i).ljust(3) + " | " + str(seconds).ljust(30) + " | " + str(table[1]).ljust(
  62. len("is_default")), end="")
  63. if len(table) > 2:
  64. print(" | " + str(table[2]))
  65. else:
  66. print("")
  67. previous_intervals.append(seconds)
  68. i = i + 1
  69. return previous_intervals
  70. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool,
  71. flag_non_verbose: bool, intervals, delete: bool=False, recalculate_intervals: bool=None):
  72. """
  73. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  74. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  75. statistics are gathered directly from the existing database.
  76. :param flag_write_file: Indicates whether the statistics should be written additionally into a text file (True)
  77. or not (False)
  78. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  79. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  80. :param flag_non_verbose: Indicates whether certain prints should be made or not, to reduce terminal clutter
  81. :param intervals: user specified interval in seconds
  82. :param delete: Delete old interval statistics.
  83. :param recalculate_intervals: Recalculate old interval statistics or not. Prompt user if None.
  84. """
  85. # Load pcap and get loading time
  86. time_start = time.clock()
  87. # Make sure user specified intervals are a list
  88. if intervals is None or intervals == []:
  89. intervals = [0.0]
  90. elif not isinstance(intervals, list):
  91. intervals = [intervals]
  92. current_intervals = intervals[:]
  93. # Inform user about recalculation of statistics and its reason
  94. if flag_recalculate_stats:
  95. print("Flag -r/--recalculate found. Recalculating statistics.")
  96. outstring_datasource = "from statistics database."
  97. # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
  98. # FIXME: probably wanna add a "calculate only extra tests" case in the future
  99. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats or self.stats_db.get_db_outdated():
  100. # Get interval statistics tables which already exist
  101. previous_intervals = self.list_previous_interval_statistic_tables()
  102. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests), Util.RESOURCE_DIR,
  103. self.path_db)
  104. recalc_intervals = None
  105. if previous_intervals:
  106. if delete:
  107. recalc_intervals = False
  108. else:
  109. recalc_intervals = recalculate_intervals
  110. while recalc_intervals is None:
  111. user_input = input("Do you want to recalculate them as well? (y)es|(n)o|(d)elete: ")
  112. if user_input.lower() == "yes" or user_input.lower() == "y":
  113. recalc_intervals = True
  114. elif user_input.lower() == "no" or user_input.lower() == "n":
  115. recalc_intervals = False
  116. elif user_input.lower() == "delete" or user_input.lower() == "d":
  117. recalc_intervals = False
  118. delete = True
  119. else:
  120. print("This was no valid input.")
  121. if recalc_intervals and previous_intervals:
  122. intervals = list(set(intervals + previous_intervals))
  123. print("The old interval statistics will be recalculated.")
  124. elif delete:
  125. print("The old interval statistics will be deleted.")
  126. else:
  127. print("The old interval statistics wont be recalculated.")
  128. if current_intervals != [0.0]:
  129. print("User specified intervals will be used to calculate interval statistics: " +
  130. str(current_intervals)[1:-1])
  131. self.pcap_proc.collect_statistics(intervals)
  132. self.pcap_proc.write_to_database(self.path_db, intervals, delete)
  133. outstring_datasource = "by PCAP file processor."
  134. # only print summary of new db if -s flag not set
  135. if not flag_print_statistics and not flag_non_verbose:
  136. self.stats_summary_new_db()
  137. elif (intervals is not None and intervals != []) or self.do_extra_tests:
  138. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests), Util.RESOURCE_DIR,
  139. self.path_db)
  140. # Get interval statistics tables which already exist
  141. previous_intervals = self.list_previous_interval_statistic_tables(output=False)
  142. final_intervals = []
  143. if not self.do_extra_tests:
  144. for interval in intervals:
  145. if interval not in previous_intervals:
  146. final_intervals.append(interval)
  147. else:
  148. final_intervals = intervals
  149. self.pcap_proc.collect_statistics(final_intervals)
  150. self.pcap_proc.write_new_interval_statistics(self.path_db, final_intervals)
  151. self.stats_db.set_current_interval_statistics_tables(current_intervals)
  152. # Load statistics from database
  153. self.file_info = self.stats_db.get_file_info()
  154. time_end = time.clock()
  155. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  156. # Write statistics if param -e/--export provided
  157. if flag_write_file:
  158. self.write_statistics_to_file()
  159. self.write_statistics_to_file(interval=True)
  160. # Print statistics if param -s/--statistics provided
  161. if flag_print_statistics:
  162. self.print_statistics()
  163. def get_file_information(self):
  164. """
  165. Returns a list of tuples, each containing a information of the file.
  166. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  167. """
  168. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  169. if pdu_count is None:
  170. pdu_count = 0
  171. pdu_share = pdu_count / self.get_packet_count() * 100
  172. last_pdu_timestamp = self.process_db_query(
  173. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  174. return [("Pcap file path", self.pcap_filepath),
  175. ("Total packet count", self.get_packet_count(), "packets"),
  176. ("Recognized packets", self.get_packet_count() - pdu_count, "packets"),
  177. ("Unrecognized packets", pdu_count, "PDUs"),
  178. ("% Recognized packets", 100 - pdu_share, "%"),
  179. ("% Unrecognized packets", pdu_share, "%"),
  180. ("Last unknown PDU", last_pdu_timestamp),
  181. ("Capture duration", self.get_capture_duration(), "seconds"),
  182. ("Capture start", "\t" + str(self.get_pcap_timestamp_start())),
  183. ("Capture end", "\t" + str(self.get_pcap_timestamp_end()))]
  184. def get_general_file_statistics(self):
  185. """
  186. Returns a list of tuples, each containing a file statistic.
  187. :return: a list of tuples, each consisting of (description, value, unit).
  188. """
  189. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  190. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  191. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  192. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  193. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  194. def get_interval_statistics(self, table_name: str=""):
  195. """
  196. Returns a list of tuples, each containing interval statistics.
  197. :param table_name: the name of the interval statistics table
  198. :return: a list of tuples, each consisting of (description, values, unit).
  199. """
  200. column_names = self.stats_db.get_field_types(table_name)
  201. column_names = sorted(column_names)
  202. result = column_names[0]
  203. for name in column_names[1:]:
  204. result += ", " + name
  205. interval_stats = self.stats_db.process_interval_statistics_query(
  206. "SELECT {} FROM %s ORDER BY first_pkt_timestamp ASC".format(result),
  207. table_name)
  208. inverted_table = {}
  209. inverted_table["interval_count"] = 0
  210. for name in column_names:
  211. inverted_table[name] = []
  212. for row in interval_stats:
  213. for column, name in zip(row, column_names):
  214. if column is not None:
  215. if type(column) == str:
  216. try:
  217. column = int(column)
  218. except ValueError:
  219. column = float(column)
  220. elif type(column) == float:
  221. column = round(column, 4)
  222. inverted_table[name].append(column)
  223. inverted_table["interval_count"] = len(inverted_table[column_names[0]])
  224. return sorted(inverted_table.items())
  225. @staticmethod
  226. def write_list(desc_val_unit_list, func, line_ending="\n"):
  227. """
  228. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three
  229. values and applies the function func on this string.
  230. Before generating the string, it identifies text containing a float number, casts the string to a
  231. float and rounds the value to two decimal digits.
  232. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  233. :param func: The function to be applied to each generated string
  234. :param line_ending: The formatting string to be applied at the end of each string
  235. """
  236. for entry in desc_val_unit_list:
  237. # Convert text containing float into float
  238. (description, value) = entry[0:2]
  239. if isinstance(value, str) and "." in value:
  240. try:
  241. value = float(value)
  242. except ValueError:
  243. pass # do nothing -> value was not a float
  244. # round float
  245. if isinstance(value, float):
  246. value = round(value, 4)
  247. # check for lists
  248. if isinstance(value, list):
  249. # remove brackets
  250. value = str(value)[1:-1]
  251. # write into file
  252. if len(entry) == 3:
  253. unit = entry[2]
  254. func(description + ":\t" + str(value) + " " + unit + line_ending)
  255. else:
  256. func(description + ":\t" + str(value) + line_ending)
  257. def print_statistics(self):
  258. """
  259. Prints the basic file statistics to the terminal.
  260. """
  261. print("\nPCAP FILE INFORMATION ------------------------------")
  262. Statistics.write_list(self.get_file_information(), print, "")
  263. print("\nGENERAL FILE STATISTICS ----------------------------")
  264. Statistics.write_list(self.get_general_file_statistics(), print, "")
  265. print("\n")
  266. @staticmethod
  267. def calculate_entropy(frequency: list, normalized: bool = False):
  268. """
  269. Calculates entropy and normalized entropy of list of elements that have specific frequency
  270. :param frequency: The frequency of the elements.
  271. :param normalized: Calculate normalized entropy
  272. :return: entropy or (entropy, normalized entropy)
  273. """
  274. entropy, normalized_ent, n = 0, 0, 0
  275. sum_freq = sum(frequency)
  276. for i, x in enumerate(frequency):
  277. p_x = float(frequency[i] / sum_freq)
  278. if p_x > 0:
  279. n += 1
  280. entropy += - p_x * log(p_x, 2)
  281. if normalized:
  282. if log(n) > 0:
  283. normalized_ent = entropy / log(n, 2)
  284. return entropy, normalized_ent
  285. else:
  286. return entropy
  287. def calculate_complement_packet_rates(self, pps):
  288. """
  289. Calculates the complement packet rates of the background traffic packet rates for each interval.
  290. Then normalize it to maximum boundary, which is the input parameter pps
  291. :return: normalized packet rates for each time interval.
  292. """
  293. result = self.stats_db.process_interval_statistics_query(
  294. "SELECT last_pkt_timestamp,pkts_count FROM %s ORDER BY last_pkt_timestamp")
  295. # print(result)
  296. bg_interval_pps = []
  297. complement_interval_pps = []
  298. intervals_sum = 0
  299. if result:
  300. # Get the interval in seconds
  301. for i, row in enumerate(result):
  302. if i < len(result) - 1:
  303. intervals_sum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
  304. interval = intervals_sum / (len(result) - 1)
  305. # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
  306. for row in result:
  307. bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
  308. # Find max PPS
  309. max_pps = max(bg_interval_pps, key=itemgetter(1))[1]
  310. for row in bg_interval_pps:
  311. complement_interval_pps.append((row[0], int(pps * (max_pps - row[1]) / max_pps)))
  312. return complement_interval_pps
  313. def get_tests_statistics(self):
  314. """
  315. Writes the calculated basic defects tests statistics into a file.
  316. """
  317. # self.stats_db.process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
  318. def count_frequncy(values_list):
  319. """
  320. TODO : FILL ME
  321. :param values_list:
  322. :return:
  323. """
  324. values, freq_output = [], []
  325. for x in values_list:
  326. if x in values:
  327. freq_output[values.index(x)] += 1
  328. else:
  329. values.append(x)
  330. freq_output.append(1)
  331. return values, freq_output
  332. # Payload Tests
  333. sum_payload_count = self.stats_db.process_interval_statistics_query("SELECT sum(payload_count) FROM %s")
  334. pkt_count = self.stats_db.process_user_defined_query("SELECT packetCount FROM file_statistics")
  335. if sum_payload_count and pkt_count:
  336. payload_ratio = 0
  337. if pkt_count[0][0] != 0:
  338. payload_ratio = float(sum_payload_count[0][0] / pkt_count[0][0] * 100)
  339. else:
  340. payload_ratio = -1
  341. # TCP checksum Tests
  342. incorrect_checksum_count = self.stats_db.process_interval_statistics_query(
  343. "SELECT sum(incorrect_tcp_checksum_count) FROM %s")
  344. correct_checksum_count = self.stats_db.process_interval_statistics_query(
  345. "SELECT avg(correct_tcp_checksum_count) FROM %s")
  346. if incorrect_checksum_count and correct_checksum_count:
  347. incorrect_checksum_ratio = 0
  348. if (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) != 0:
  349. incorrect_checksum_ratio = float(incorrect_checksum_count[0][0] /
  350. (incorrect_checksum_count[0][0] + correct_checksum_count[0][0]) * 100)
  351. else:
  352. incorrect_checksum_ratio = -1
  353. # IP Src & Dst Tests
  354. result = self.stats_db.process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
  355. data, src_frequency, dst_frequency = [], [], []
  356. if result:
  357. for row in result:
  358. src_frequency.append(row[1])
  359. dst_frequency.append(row[2])
  360. ip_src_entropy, ip_src_norm_entropy = self.calculate_entropy(src_frequency, True)
  361. ip_dst_entropy, ip_dst_norm_entropy = self.calculate_entropy(dst_frequency, True)
  362. new_ip_src_count = self.stats_db.process_interval_statistics_query("SELECT ip_src_novel_Count FROM %s")
  363. ip_src_novels_per_interval, ip_src_novels_per_interval_frequency = count_frequncy(new_ip_src_count)
  364. ip_src_novelty_dist_entropy = self.calculate_entropy(ip_src_novels_per_interval_frequency)
  365. new_ip_dst_count = self.stats_db.process_interval_statistics_query("SELECT ip_dst_novel_Count FROM %s")
  366. ip_dst_novels_per_interval, ip_dst_novels_per_interval_frequency = count_frequncy(new_ip_dst_count)
  367. ip_dst_novelty_dist_entropy = self.calculate_entropy(ip_dst_novels_per_interval_frequency)
  368. # Ports Tests
  369. port0_count = self.stats_db.process_user_defined_query(
  370. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
  371. if not port0_count[0][0]:
  372. port0_count = 0
  373. else:
  374. port0_count = port0_count[0][0]
  375. # FIXME: could be extended
  376. reserved_port_count = self.stats_db.process_user_defined_query(
  377. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")
  378. if not reserved_port_count[0][0]:
  379. reserved_port_count = 0
  380. else:
  381. reserved_port_count = reserved_port_count[0][0]
  382. # TTL Tests
  383. result = self.stats_db.process_user_defined_query(
  384. "SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  385. data, frequency = [], []
  386. for row in result:
  387. frequency.append(row[1])
  388. ttl_entropy, ttl_norm_entropy = self.calculate_entropy(frequency, True)
  389. new_ttl_count = self.stats_db.process_interval_statistics_query("SELECT ttl_novel_count FROM %s")
  390. ttl_novels_per_interval, ttl_novels_per_interval_frequency = count_frequncy(new_ttl_count)
  391. ttl_novelty_dist_entropy = self.calculate_entropy(ttl_novels_per_interval_frequency)
  392. # Window Size Tests
  393. result = self.stats_db.process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
  394. data, frequency = [], []
  395. for row in result:
  396. frequency.append(row[1])
  397. win_entropy, win_norm_entropy = self.calculate_entropy(frequency, True)
  398. new_win_size_count = self.stats_db.process_interval_statistics_query("SELECT win_size_novel_count FROM %s")
  399. win_novels_per_interval, win_novels_per_interval_frequency = count_frequncy(new_win_size_count)
  400. win_novelty_dist_entropy = self.calculate_entropy(win_novels_per_interval_frequency)
  401. # ToS Tests
  402. result = self.stats_db.process_user_defined_query(
  403. "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
  404. data, frequency = [], []
  405. for row in result:
  406. frequency.append(row[1])
  407. tos_entropy, tos_norm_entropy = self.calculate_entropy(frequency, True)
  408. new_tos_count = self.stats_db.process_interval_statistics_query("SELECT tos_novel_count FROM %s")
  409. tos_novels_per_interval, tos_novels_per_interval_frequency = count_frequncy(new_tos_count)
  410. tos_novelty_dist_entropy = self.calculate_entropy(tos_novels_per_interval_frequency)
  411. # MSS Tests
  412. result = self.stats_db.process_user_defined_query(
  413. "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  414. data, frequency = [], []
  415. for row in result:
  416. frequency.append(row[1])
  417. mss_entropy, mss_norm_entropy = self.calculate_entropy(frequency, True)
  418. new_mss_count = self.stats_db.process_interval_statistics_query("SELECT mss_novel_count FROM %s")
  419. mss_novels_per_interval, mss_novels_per_interval_frequency = count_frequncy(new_mss_count)
  420. mss_novelty_dist_entropy = self.calculate_entropy(mss_novels_per_interval_frequency)
  421. result = self.stats_db.process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
  422. # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
  423. if not result[0][0]:
  424. result = 0
  425. else:
  426. result = result[0][0]
  427. big_mss = (result / sum(frequency)) * 100
  428. output = []
  429. if self.do_extra_tests:
  430. output = [("Payload ratio", payload_ratio, "%"),
  431. ("Incorrect TCP checksum ratio", incorrect_checksum_ratio, "%")]
  432. output = output + [("# IP addresses", sum([x[0] for x in new_ip_src_count]), ""),
  433. ("IP Src Entropy", ip_src_entropy, ""),
  434. ("IP Src Normalized Entropy", ip_src_norm_entropy, ""),
  435. ("IP Dst Entropy", ip_dst_entropy, ""),
  436. ("IP Dst Normalized Entropy", ip_dst_norm_entropy, ""),
  437. ("IP SRC Novelty Distribution Entropy", ip_src_novelty_dist_entropy, ""),
  438. ("IP DST Novelty Distribution Entropy", ip_dst_novelty_dist_entropy, ""),
  439. ("# TTL values", sum([x[0] for x in new_ttl_count]), ""),
  440. ("TTL Entropy", ttl_entropy, ""),
  441. ("TTL Normalized Entropy", ttl_norm_entropy, ""),
  442. ("TTL Novelty Distribution Entropy", ttl_novelty_dist_entropy, ""),
  443. ("# WinSize values", sum([x[0] for x in new_win_size_count]), ""),
  444. ("WinSize Entropy", win_entropy, ""),
  445. ("WinSize Normalized Entropy", win_norm_entropy, ""),
  446. ("WinSize Novelty Distribution Entropy", win_novelty_dist_entropy, ""),
  447. ("# ToS values", sum([x[0] for x in new_tos_count]), ""),
  448. ("ToS Entropy", tos_entropy, ""),
  449. ("ToS Normalized Entropy", tos_norm_entropy, ""),
  450. ("ToS Novelty Distribution Entropy", tos_novelty_dist_entropy, ""),
  451. ("# MSS values", sum([x[0] for x in new_mss_count]), ""),
  452. ("MSS Entropy", mss_entropy, ""),
  453. ("MSS Normalized Entropy", mss_norm_entropy, ""),
  454. ("MSS Novelty Distribution Entropy", mss_novelty_dist_entropy, ""),
  455. ("======================", "", "")]
  456. # Reasoning the statistics values
  457. if self.do_extra_tests:
  458. if payload_ratio > 80:
  459. output.append(("WARNING: Too high payload ratio", payload_ratio, "%."))
  460. if payload_ratio < 30:
  461. output.append(("WARNING: Too low payload ratio", payload_ratio, "% (Injecting attacks that are carried "
  462. "out in the packet payloads is not "
  463. "recommmanded)."))
  464. if incorrect_checksum_ratio > 5:
  465. output.append(("WARNING: High incorrect TCP checksum ratio", incorrect_checksum_ratio, "%."))
  466. if ip_src_norm_entropy > 0.65:
  467. output.append(("WARNING: High IP source normalized entropy", ip_src_norm_entropy, "."))
  468. if ip_src_norm_entropy < 0.2:
  469. output.append(("WARNING: Low IP source normalized entropy", ip_src_norm_entropy, "."))
  470. if ip_dst_norm_entropy > 0.65:
  471. output.append(("WARNING: High IP destination normalized entropy", ip_dst_norm_entropy, "."))
  472. if ip_dst_norm_entropy < 0.2:
  473. output.append(("WARNING: Low IP destination normalized entropy", ip_dst_norm_entropy, "."))
  474. if ttl_norm_entropy > 0.65:
  475. output.append(("WARNING: High TTL normalized entropy", ttl_norm_entropy, "."))
  476. if ttl_norm_entropy < 0.2:
  477. output.append(("WARNING: Low TTL normalized entropy", ttl_norm_entropy, "."))
  478. if ttl_novelty_dist_entropy < 1:
  479. output.append(("WARNING: Too low TTL novelty distribution entropy", ttl_novelty_dist_entropy,
  480. "(The distribution of the novel TTL values is suspicious)."))
  481. if win_norm_entropy > 0.6:
  482. output.append(("WARNING: High Window Size normalized entropy", win_norm_entropy, "."))
  483. if win_norm_entropy < 0.1:
  484. output.append(("WARNING: Low Window Size normalized entropy", win_norm_entropy, "."))
  485. if win_novelty_dist_entropy < 4:
  486. output.append(("WARNING: Low Window Size novelty distribution entropy", win_novelty_dist_entropy,
  487. "(The distribution of the novel Window Size values is suspicious)."))
  488. if tos_norm_entropy > 0.4:
  489. output.append(("WARNING: High ToS normalized entropy", tos_norm_entropy, "."))
  490. if tos_norm_entropy < 0.1:
  491. output.append(("WARNING: Low ToS normalized entropy", tos_norm_entropy, "."))
  492. if tos_novelty_dist_entropy < 0.5:
  493. output.append(("WARNING: Low ToS novelty distribution entropy", tos_novelty_dist_entropy,
  494. "(The distribution of the novel ToS values is suspicious)."))
  495. if mss_norm_entropy > 0.4:
  496. output.append(("WARNING: High MSS normalized entropy", mss_norm_entropy, "."))
  497. if mss_norm_entropy < 0.1:
  498. output.append(("WARNING: Low MSS normalized entropy", mss_norm_entropy, "."))
  499. if mss_novelty_dist_entropy < 0.5:
  500. output.append(("WARNING: Low MSS novelty distribution entropy", mss_novelty_dist_entropy,
  501. "(The distribution of the novel MSS values is suspicious)."))
  502. if big_mss > 50:
  503. output.append(("WARNING: High ratio of MSS > 1460", big_mss, "% (High fragmentation rate in Ethernet)."))
  504. if port0_count > 0:
  505. output.append(("WARNING: Port number 0 is used in ", port0_count, "packets (awkward-looking port)."))
  506. if reserved_port_count > 0:
  507. output.append(("WARNING: Reserved port numbers are used in ", reserved_port_count,
  508. "packets (uncommonly-used ports)."))
  509. return output
  510. def write_statistics_to_file(self, interval: bool=False):
  511. """
  512. Writes the calculated basic statistics into a file.
  513. """
  514. def _write_header(title: str):
  515. """
  516. Writes the section header into the open file.
  517. :param title: The section title
  518. """
  519. target.write("====================== \n")
  520. target.write(title + " \n")
  521. target.write("====================== \n")
  522. def _write_sub_header(title: str):
  523. """
  524. Writes the section header into the open file.
  525. :param title: The section title
  526. """
  527. target.write("---------------------- \n")
  528. target.write(title + " \n")
  529. target.write("---------------------- \n")
  530. if interval:
  531. ed = ".interval_stat"
  532. else:
  533. ed = ".stat"
  534. target = open(self.pcap_filepath + ed, 'w')
  535. target.truncate()
  536. _write_header("PCAP file information")
  537. Statistics.write_list(self.get_file_information(), target.write)
  538. if interval:
  539. _write_header("Interval statistics")
  540. tables = self.stats_db.get_all_current_interval_statistics_tables()
  541. for table in tables:
  542. _write_sub_header(table[len("interval_staistiscs_"):] + " microseconds")
  543. Statistics.write_list(self.get_interval_statistics(table), target.write)
  544. else:
  545. _write_header("General statistics")
  546. Statistics.write_list(self.get_general_file_statistics(), target.write)
  547. _write_header("Tests statistics")
  548. Statistics.write_list(self.get_tests_statistics(), target.write)
  549. target.close()
  550. def get_capture_duration(self):
  551. """
  552. :return: The duration of the capture in seconds
  553. """
  554. return self.file_info['captureDuration']
  555. def get_pcap_timestamp_start(self):
  556. """
  557. :return: The timestamp of the first packet in the PCAP file
  558. """
  559. return self.file_info['timestampFirstPacket']
  560. def get_pcap_timestamp_end(self):
  561. """
  562. :return: The timestamp of the last packet in the PCAP file
  563. """
  564. return self.file_info['timestampLastPacket']
  565. def get_pps_sent(self, ip_address: str):
  566. """
  567. Calculates the sent packets per seconds for a given IP address.
  568. :param ip_address: The IP address whose packets per second should be calculated
  569. :return: The sent packets per seconds for the given IP address
  570. """
  571. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  572. (ip_address,))
  573. capture_duration = float(self.get_capture_duration())
  574. return int(float(packets_sent) / capture_duration)
  575. def get_pps_received(self, ip_address: str):
  576. """
  577. Calculate the packets per second received for a given IP address.
  578. :param ip_address: The IP address used for the calculation
  579. :return: The number of packets per second received
  580. """
  581. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  582. False,
  583. (ip_address,))
  584. capture_duration = float(self.get_capture_duration())
  585. return int(float(packets_received) / capture_duration)
  586. def get_packet_count(self):
  587. """
  588. :return: The number of packets in the loaded PCAP file
  589. """
  590. return self.file_info['packetCount']
  591. def get_most_used_ip_address(self):
  592. """
  593. :return: The IP address/addresses with the highest sum of packets sent and received
  594. """
  595. return Util.handle_most_used_outputs(self.process_db_query("most_used(ipAddress)"))
  596. def get_ttl_distribution(self, ip_address: str):
  597. """
  598. TODO: FILL ME
  599. :param ip_address:
  600. :return:
  601. """
  602. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ip_address + '"')
  603. result_dict = {key: value for (key, value) in result}
  604. return result_dict
  605. def get_mss_distribution(self, ip_address: str):
  606. """
  607. TODO: FILL ME
  608. :param ip_address:
  609. :return:
  610. """
  611. result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ip_address + '"')
  612. result_dict = {key: value for (key, value) in result}
  613. return result_dict
  614. def get_win_distribution(self, ip_address: str):
  615. """
  616. TODO: FILL ME
  617. :param ip_address:
  618. :return:
  619. """
  620. result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ip_address + '"')
  621. result_dict = {key: value for (key, value) in result}
  622. return result_dict
  623. def get_tos_distribution(self, ip_address: str):
  624. """
  625. TODO: FILL ME
  626. :param ip_address:
  627. :return:
  628. """
  629. result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ip_address + '"')
  630. result_dict = {key: value for (key, value) in result}
  631. return result_dict
  632. def get_ip_address_count(self):
  633. """
  634. TODO: FILL ME
  635. :return:
  636. """
  637. return self.process_db_query("SELECT COUNT(*) FROM ip_statistics")
  638. def get_ip_addresses(self):
  639. """
  640. TODO: FILL ME
  641. :return:
  642. """
  643. return self.process_db_query("SELECT ipAddress FROM ip_statistics")
  644. def get_random_ip_address(self, count: int = 1):
  645. """
  646. :param count: The number of IP addresses to return
  647. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of
  648. randomly chosen IP addresses
  649. """
  650. ip_address_list = self.process_db_query("SELECT ipAddress from ip_statistics ORDER BY ipAddress ASC")
  651. if count == 1:
  652. return random.choice(ip_address_list)
  653. else:
  654. result_list = []
  655. for i in range(0, count):
  656. random_ip = random.choice(ip_address_list)
  657. result_list.append(random_ip)
  658. ip_address_list.remove(random_ip)
  659. return result_list
  660. def get_ip_address_from_mac(self, mac_address: str):
  661. """
  662. :param mac_address: the MAC address of which the IP shall be returned, if existing in DB
  663. :return: the IP address used in the dataset by a given MAC address
  664. """
  665. return self.process_db_query("SELECT DISTINCT ipAddress FROM ip_mac WHERE macAddress = '" + mac_address + "'")
  666. def get_mac_addresses(self, ip_addresses: list):
  667. """
  668. :return: The MAC addresses used in the dataset for the given IP addresses as a dictionary.
  669. """
  670. return dict(self.process_db_query("SELECT DISTINCT ipAddress, macAddress from ip_mac WHERE ipAddress in ("
  671. + str(ip_addresses)[1:-1] + ")"))
  672. def get_mac_address(self, ip_address: str):
  673. """
  674. :return: The MAC address used in the dataset for the given IP address.
  675. """
  676. return self.process_db_query("SELECT DISTINCT macAddress from ip_mac WHERE ipAddress = '" + ip_address + "'")
  677. def get_most_used_ttl_value(self):
  678. """
  679. :return: The most used TTL value.
  680. """
  681. return self.process_db_query("SELECT ttlValue FROM (SELECT ttlValue, SUM(ttlCount) as occ FROM ip_ttl GROUP BY "
  682. "ttlValue) WHERE occ=(SELECT SUM(ttlCount) as occ FROM ip_ttl GROUP BY ttlValue "
  683. "ORDER BY occ DESC LIMIT 1) ORDER BY ttlValue ASC")
  684. def get_most_used_ip_class(self):
  685. """
  686. :return: The most used IP class.
  687. """
  688. return self.process_db_query("SELECT ipClass FROM (SELECT ipClass, COUNT(*) as occ from ip_statistics GROUP BY "
  689. "ipClass ORDER BY occ DESC) WHERE occ=(SELECT COUNT(*) as occ from ip_statistics "
  690. "GROUP BY ipClass ORDER BY occ DESC LIMIT 1) ORDER BY ipClass ASC")
  691. def get_most_used_win_size(self):
  692. """
  693. :return: The most used window size.
  694. """
  695. return self.process_db_query("SELECT winSize FROM (SELECT winSize, SUM(winCount) as occ FROM tcp_win GROUP BY "
  696. "winSize) WHERE occ=(SELECT SUM(winCount) as occ FROM tcp_win GROUP BY winSize "
  697. "ORDER BY occ DESC LIMIT 1) ORDER BY winSize ASC")
  698. def get_most_used_mss_value(self):
  699. """
  700. :return: The most used mss value.
  701. """
  702. return self.process_db_query("SELECT mssValue FROM (SELECT mssValue, SUM(mssCount) as occ FROM tcp_mss GROUP BY"
  703. " mssValue) WHERE occ=(SELECT SUM(mssCount) as occ FROM tcp_mss GROUP BY mssValue "
  704. "ORDER BY occ DESC LIMIT 1) ORDER BY mssValue ASC")
  705. def get_most_used_mss(self, ip_address: str):
  706. """
  707. :param ip_address: The IP address whose used MSS should be determined
  708. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  709. then None is returned
  710. """
  711. mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ip_address +
  712. '" AND mssCount == (SELECT MAX(mssCount) from tcp_mss WHERE ipAddress="'
  713. + ip_address + '")')
  714. if isinstance(mss_value, int):
  715. return mss_value
  716. elif isinstance(mss_value, list):
  717. if len(mss_value) == 0:
  718. return None
  719. else:
  720. mss_value.sort()
  721. return mss_value[0]
  722. else:
  723. return None
  724. def get_most_used_ttl(self, ip_address: str):
  725. """
  726. :param ip_address: The IP address whose used TTL should be determined
  727. :return: The TTL value used by the IP address, or if the IP addresses never specified a TTL,
  728. then None is returned
  729. """
  730. ttl_value = self.process_db_query('SELECT ttlValue from ip_ttl WHERE ipAddress="' + ip_address +
  731. '" AND ttlCount == (SELECT MAX(ttlCount) from ip_ttl WHERE ipAddress="'
  732. + ip_address + '")')
  733. if isinstance(ttl_value, int):
  734. return ttl_value
  735. elif isinstance(ttl_value, list):
  736. if len(ttl_value) == 0:
  737. return None
  738. else:
  739. ttl_value.sort()
  740. return ttl_value[0]
  741. else:
  742. return None
  743. def get_avg_delay_distributions(self, input_pcap: bool=True):
  744. """
  745. :return: tuple consisting of avg delay distributions for local and external communication
  746. """
  747. if input_pcap:
  748. delay_db = self.stats_db
  749. else:
  750. delay_db = statsDB.StatsDatabase(Util.get_botnet_pcap_db())
  751. conv_delays = delay_db.process_user_defined_query(
  752. "SELECT ipAddressA, ipAddressB, avgDelay FROM conv_statistics")
  753. conv_delays += delay_db.process_user_defined_query(
  754. "SELECT ipAddressA, ipAddressB, avgDelay FROM conv_statistics_extended")
  755. if conv_delays:
  756. external_conv = []
  757. local_conv = []
  758. for conv in conv_delays:
  759. if conv[2] is not None:
  760. ip_a = IPAddress.parse(conv[0])
  761. ip_b = IPAddress.parse(conv[1])
  762. # split into local and external conversations
  763. if not ip_a.is_private() or not ip_b.is_private():
  764. external_conv.append(conv)
  765. else:
  766. local_conv.append(conv)
  767. local_dist = []
  768. for conv in local_conv:
  769. local_dist.append(conv[2])
  770. external_dist = []
  771. for conv in external_conv:
  772. external_dist.append(conv[2])
  773. return local_dist, external_dist
  774. return [], []
  775. def get_filtered_degree(self, degree_type: str):
  776. """
  777. gets the desired type of degree statistics and filters IPs with degree value zero
  778. :param degree_type: the desired type of degrees, one of the following: inDegree, outDegree, overallDegree
  779. :return: the filtered degrees
  780. """
  781. degrees_raw = self.stats_db.process_user_defined_query(
  782. "SELECT ipAddress, %s FROM ip_degrees" % degree_type)
  783. degrees = []
  784. if degrees_raw:
  785. for deg in degrees_raw:
  786. if int(deg[1]) > 0:
  787. degrees.append(deg)
  788. return degrees
  789. def get_rnd_win_size(self, pkts_num):
  790. """
  791. :param pkts_num: maximum number of window sizes, that should be returned
  792. :return: A list of randomly chosen window sizes with given length.
  793. """
  794. sql_return = self.process_db_query("SELECT DISTINCT winSize FROM tcp_win ORDER BY winsize ASC;")
  795. if not isinstance(sql_return, list):
  796. return [sql_return]
  797. result = []
  798. for i in range(0, min(pkts_num, len(sql_return))):
  799. result.append(random.choice(sql_return))
  800. sql_return.remove(result[i])
  801. return result
  802. def get_statistics_database(self):
  803. """
  804. :return: A reference to the statistics database object
  805. """
  806. return self.stats_db
  807. def process_db_query(self, query_string_in: str, print_results: bool = False):
  808. """
  809. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  810. query.
  811. :param query_string_in: The query to be processed
  812. :param print_results: Indicates whether the results should be printed to terminal
  813. :return: The result of the query
  814. """
  815. return self.stats_db.process_db_query(query_string_in, print_results)
  816. def is_query(self, value: str):
  817. """
  818. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  819. :param value: The string to be checked
  820. :return: True if the string is recognized as a query, otherwise False.
  821. """
  822. if not isinstance(value, str):
  823. return False
  824. else:
  825. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  826. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  827. @staticmethod
  828. def calculate_standard_deviation(lst):
  829. """
  830. Calculates the standard deviation of a list of numbers.
  831. :param lst: The list of numbers to calculate its SD.
  832. """
  833. num_items = len(lst)
  834. mean = sum(lst) / num_items
  835. differences = [x - mean for x in lst]
  836. sq_differences = [d ** 2 for d in differences]
  837. ssd = sum(sq_differences)
  838. variance = ssd / num_items
  839. sd = sqrt(variance)
  840. return sd
  841. def plot_statistics(self, entropy: int, file_format: str = 'pdf'): # 'png'
  842. """
  843. Plots the statistics associated with the dataset.
  844. :param entropy: the statistics entropy
  845. :param file_format: The format to be used to save the statistics diagrams.
  846. """
  847. def plot_distribution(query_output, title, x_label, y_label, file_ending: str):
  848. """
  849. TODO: FILL ME
  850. :param query_output:
  851. :param title:
  852. :param x_label:
  853. :param y_label:
  854. :param file_ending:
  855. :return:
  856. """
  857. plt.gcf().clear()
  858. graphx, graphy = [], []
  859. for row in query_output:
  860. graphx.append(row[0])
  861. graphy.append(row[1])
  862. plt.autoscale(enable=True, axis='both')
  863. plt.title(title)
  864. plt.xlabel(x_label)
  865. plt.ylabel(y_label)
  866. width = 0.1
  867. plt.xlim([0, (max(graphx) * 1.1)])
  868. plt.grid(True)
  869. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  870. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  871. plt.savefig(out, dpi=500)
  872. return out
  873. def plot_ttl(file_ending: str):
  874. """
  875. TODO: FILL ME
  876. :param file_ending:
  877. :return:
  878. """
  879. query_output = self.stats_db.process_user_defined_query(
  880. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  881. title = "TTL Distribution"
  882. x_label = "TTL Value"
  883. y_label = "Number of Packets"
  884. if query_output:
  885. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  886. def plot_mss(file_ending: str):
  887. """
  888. TODO: FILL ME
  889. :param file_ending:
  890. :return:
  891. """
  892. query_output = self.stats_db.process_user_defined_query(
  893. "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  894. title = "MSS Distribution"
  895. x_label = "MSS Value"
  896. y_label = "Number of Packets"
  897. if query_output:
  898. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  899. def plot_win(file_ending: str):
  900. """
  901. TODO: FILL ME
  902. :param file_ending:
  903. :return:
  904. """
  905. query_output = self.stats_db.process_user_defined_query(
  906. "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
  907. title = "Window Size Distribution"
  908. x_label = "Window Size"
  909. y_label = "Number of Packets"
  910. if query_output:
  911. return plot_distribution(query_output, title, x_label, y_label, file_ending)
  912. def plot_protocol(file_ending: str):
  913. """
  914. TODO: FILL ME
  915. :param file_ending:
  916. :return:
  917. """
  918. plt.gcf().clear()
  919. result = self.stats_db.process_user_defined_query(
  920. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  921. if result:
  922. graphx, graphy = [], []
  923. for row in result:
  924. graphx.append(row[0])
  925. graphy.append(row[1])
  926. plt.autoscale(enable=True, axis='both')
  927. plt.title("Protocols Distribution")
  928. plt.xlabel('Protocols')
  929. plt.ylabel('Number of Packets')
  930. width = 0.1
  931. plt.xlim([0, len(graphx)])
  932. plt.grid(True)
  933. # Protocols' names on x-axis
  934. x = range(0, len(graphx))
  935. my_xticks = graphx
  936. plt.xticks(x, my_xticks)
  937. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  938. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  939. plt.savefig(out, dpi=500)
  940. return out
  941. else:
  942. print("Error plot protocol: No protocol values found!")
  943. def plot_port(file_ending: str):
  944. """
  945. TODO: FILL ME
  946. :param file_ending:
  947. :return:
  948. """
  949. plt.gcf().clear()
  950. result = self.stats_db.process_user_defined_query(
  951. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  952. graphx, graphy = [], []
  953. for row in result:
  954. graphx.append(row[0])
  955. graphy.append(row[1])
  956. plt.autoscale(enable=True, axis='both')
  957. plt.title("Ports Distribution")
  958. plt.xlabel('Ports Numbers')
  959. plt.ylabel('Number of Packets')
  960. width = 0.1
  961. plt.xlim([0, max(graphx)])
  962. plt.grid(True)
  963. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  964. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  965. plt.savefig(out, dpi=500)
  966. return out
  967. # This distribution is not drawable for big datasets
  968. def plot_ip_src(file_ending: str):
  969. """
  970. TODO: FILL ME
  971. :param file_ending:
  972. :return:
  973. """
  974. plt.gcf().clear()
  975. result = self.stats_db.process_user_defined_query(
  976. "SELECT ipAddress, pktsSent FROM ip_statistics")
  977. graphx, graphy = [], []
  978. for row in result:
  979. graphx.append(row[0])
  980. graphy.append(row[1])
  981. plt.autoscale(enable=True, axis='both')
  982. plt.title("Source IP Distribution")
  983. plt.xlabel('Source IP')
  984. plt.ylabel('Number of Packets')
  985. width = 0.1
  986. plt.xlim([0, len(graphx)])
  987. plt.grid(True)
  988. # IPs on x-axis
  989. x = range(0, len(graphx))
  990. my_xticks = graphx
  991. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  992. # plt.tight_layout()
  993. # limit the number of xticks
  994. plt.locator_params(axis='x', nbins=20)
  995. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  996. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  997. plt.savefig(out, dpi=500)
  998. return out
  999. # This distribution is not drawable for big datasets
  1000. def plot_ip_dst(file_ending: str):
  1001. """
  1002. TODO: FILL ME
  1003. :param file_ending:
  1004. :return:
  1005. """
  1006. plt.gcf().clear()
  1007. result = self.stats_db.process_user_defined_query(
  1008. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  1009. graphx, graphy = [], []
  1010. for row in result:
  1011. graphx.append(row[0])
  1012. graphy.append(row[1])
  1013. plt.autoscale(enable=True, axis='both')
  1014. plt.title("Destination IP Distribution")
  1015. plt.xlabel('Destination IP')
  1016. plt.ylabel('Number of Packets')
  1017. width = 0.1
  1018. plt.xlim([0, len(graphx)])
  1019. plt.grid(True)
  1020. # IPs on x-axis
  1021. x = range(0, len(graphx))
  1022. my_xticks = graphx
  1023. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  1024. # plt.tight_layout()
  1025. # limit the number of xticks
  1026. plt.locator_params(axis='x', nbins=20)
  1027. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  1028. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  1029. plt.savefig(out, dpi=500)
  1030. return out
  1031. def plot_interval_statistics(query_output, title, x_label, y_label, file_ending: str):
  1032. """
  1033. TODO: FILL ME
  1034. :param query_output:
  1035. :param title:
  1036. :param x_label:
  1037. :param y_label:
  1038. :param file_ending:
  1039. :return:
  1040. """
  1041. plt.gcf().clear()
  1042. graphx, graphy = [], []
  1043. for row in query_output:
  1044. graphx.append(row[0])
  1045. graphy.append(row[1])
  1046. plt.autoscale(enable=True, axis='both')
  1047. plt.title(title)
  1048. plt.xlabel(x_label)
  1049. plt.ylabel(y_label)
  1050. width = 0.5
  1051. plt.xlim([0, len(graphx)])
  1052. plt.grid(True)
  1053. # timestamp on x-axis
  1054. x = range(0, len(graphx))
  1055. # limit the number of xticks
  1056. plt.locator_params(axis='x', nbins=20)
  1057. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  1058. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  1059. plt.savefig(out, dpi=500)
  1060. return out
  1061. def plot_interval_pkt_count(file_ending: str):
  1062. """
  1063. TODO: FILL ME
  1064. :param file_ending:
  1065. :return:
  1066. """
  1067. query_output = self.stats_db.process_interval_statistics_query(
  1068. "SELECT last_pkt_timestamp, pkts_count FROM %s ORDER BY last_pkt_timestamp")
  1069. title = "Packet Rate"
  1070. x_label = "Time Interval"
  1071. y_label = "Number of Packets"
  1072. if query_output:
  1073. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1074. def plot_interval_ip_src_ent(file_ending: str):
  1075. """
  1076. TODO: FILL ME
  1077. :param file_ending:
  1078. :return:
  1079. """
  1080. query_output = self.stats_db.process_interval_statistics_query(
  1081. "SELECT last_pkt_timestamp, ip_src_entropy FROM %s ORDER BY last_pkt_timestamp")
  1082. title = "Source IP Entropy"
  1083. x_label = "Time Interval"
  1084. y_label = "Entropy"
  1085. if query_output:
  1086. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1087. def plot_interval_ip_dst_ent(file_ending: str):
  1088. """
  1089. TODO: FILL ME
  1090. :param file_ending:
  1091. :return:
  1092. """
  1093. query_output = self.stats_db.process_interval_statistics_query(
  1094. "SELECT last_pkt_timestamp, ip_dst_entropy FROM %s ORDER BY last_pkt_timestamp")
  1095. title = "Destination IP Entropy"
  1096. x_label = "Time Interval"
  1097. y_label = "Entropy"
  1098. if query_output:
  1099. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1100. def plot_interval_new_ip(file_ending: str):
  1101. """
  1102. TODO: FILL ME
  1103. :param file_ending:
  1104. :return:
  1105. """
  1106. query_output = self.stats_db.process_interval_statistics_query(
  1107. "SELECT last_pkt_timestamp, newIPCount FROM %s ORDER BY last_pkt_timestamp")
  1108. title = "IP Novelty Distribution"
  1109. x_label = "Time Interval"
  1110. y_label = "Novel values count"
  1111. if query_output:
  1112. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1113. def plot_interval_new_port(file_ending: str):
  1114. """
  1115. TODO: FILL ME
  1116. :param file_ending:
  1117. :return:
  1118. """
  1119. query_output = self.stats_db.process_interval_statistics_query(
  1120. "SELECT last_pkt_timestamp, port_novel_count FROM %s ORDER BY last_pkt_timestamp")
  1121. title = "Port Novelty Distribution"
  1122. x_label = "Time Interval"
  1123. y_label = "Novel values count"
  1124. if query_output:
  1125. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1126. def plot_interval_new_ttl(file_ending: str):
  1127. """
  1128. TODO: FILL ME
  1129. :param file_ending:
  1130. :return:
  1131. """
  1132. query_output = self.stats_db.process_interval_statistics_query(
  1133. "SELECT last_pkt_timestamp, ttl_novel_count FROM %s ORDER BY last_pkt_timestamp")
  1134. title = "TTL Novelty Distribution"
  1135. x_label = "Time Interval"
  1136. y_label = "Novel values count"
  1137. if query_output:
  1138. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1139. def plot_interval_new_tos(file_ending: str):
  1140. """
  1141. TODO: FILL ME
  1142. :param file_ending:
  1143. :return:
  1144. """
  1145. query_output = self.stats_db.process_interval_statistics_query(
  1146. "SELECT last_pkt_timestamp, tos_novel_count FROM %s ORDER BY last_pkt_timestamp")
  1147. title = "ToS Novelty Distribution"
  1148. x_label = "Time Interval"
  1149. y_label = "Novel values count"
  1150. if query_output:
  1151. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1152. def plot_interval_new_win_size(file_ending: str):
  1153. """
  1154. TODO: FILL ME
  1155. :param file_ending:
  1156. :return:
  1157. """
  1158. query_output = self.stats_db.process_interval_statistics_query(
  1159. "SELECT last_pkt_timestamp, win_size_novel_count FROM %s ORDER BY last_pkt_timestamp")
  1160. title = "Window Size Novelty Distribution"
  1161. x_label = "Time Interval"
  1162. y_label = "Novel values count"
  1163. if query_output:
  1164. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1165. def plot_interval_new_mss(file_ending: str):
  1166. """
  1167. TODO: FILL ME
  1168. :param file_ending:
  1169. :return:
  1170. """
  1171. query_output = self.stats_db.process_interval_statistics_query(
  1172. "SELECT last_pkt_timestamp, mss_novel_count FROM %s ORDER BY last_pkt_timestamp")
  1173. title = "MSS Novelty Distribution"
  1174. x_label = "Time Interval"
  1175. y_label = "Novel values count"
  1176. if query_output:
  1177. return plot_interval_statistics(query_output, title, x_label, y_label, file_ending)
  1178. def plot_interval_ip_cum_ent(ip_type: str, file_ending: str):
  1179. """
  1180. TODO: FILL ME
  1181. :param ip_type: source or destination
  1182. :param file_ending:
  1183. :return:
  1184. """
  1185. if ip_type is "src":
  1186. sod = "Src"
  1187. full = "Source"
  1188. elif ip_type is "dst":
  1189. sod = "Dst"
  1190. full = "Destination"
  1191. else:
  1192. return None
  1193. plt.gcf().clear()
  1194. result = self.stats_db.process_interval_statistics_query(
  1195. "SELECT last_pkt_timestamp, ip{0}_cum_entropy FROM %s ORDER BY last_pkt_timestamp".format(sod))
  1196. graphx, graphy = [], []
  1197. for row in result:
  1198. graphx.append(row[0])
  1199. graphy.append(row[1])
  1200. # If entropy was not calculated do not plot the graph
  1201. if graphy[0] != -1:
  1202. plt.autoscale(enable=True, axis='both')
  1203. plt.title(full + " IP Cumulative Entropy")
  1204. # plt.xlabel('Timestamp')
  1205. plt.xlabel('Time Interval')
  1206. plt.ylabel('Entropy')
  1207. plt.xlim([0, len(graphx)])
  1208. plt.grid(True)
  1209. # timestamp on x-axis
  1210. x = range(0, len(graphx))
  1211. # my_xticks = graphx
  1212. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  1213. # plt.tight_layout()
  1214. # limit the number of xticks
  1215. plt.locator_params(axis='x', nbins=20)
  1216. plt.plot(x, graphy, 'r')
  1217. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-' + ip_type + '-cum-ent' + file_ending)
  1218. plt.savefig(out, dpi=500)
  1219. return out
  1220. def plot_degree(degree_type: str, file_ending: str):
  1221. """
  1222. Creates a Plot, visualizing a degree for every IP Address
  1223. :param degree_type: the type of degree, which should be plotted
  1224. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  1225. :return: A filepath to the file containing the created plot
  1226. """
  1227. if degree_type not in ["in", "out", "overall"]:
  1228. return None
  1229. plt.gcf().clear()
  1230. # retrieve data
  1231. degree = self.stats_db.process_user_defined_query(
  1232. "SELECT ipAddress, %s FROM ip_degrees" % (degree_type + "Degree"))
  1233. if degree is None:
  1234. return None
  1235. graphx, graphy = [], []
  1236. for entry in degree:
  1237. if entry[1] <= 0:
  1238. continue
  1239. # degree values
  1240. graphx.append(entry[1])
  1241. # IP labels
  1242. graphy.append(entry[0])
  1243. # set labels
  1244. plt.title(degree_type + " Degree per IP Address")
  1245. plt.ylabel('IpAddress')
  1246. plt.xlabel(degree_type + 'Degree')
  1247. # set width of the bars
  1248. width = 0.3
  1249. # set scalings
  1250. plt.figure(
  1251. figsize=(int(len(graphx)) / 20 + 5, int(len(graphy) / 5) + 5)) # these proportions just worked well
  1252. # set limits of the axis
  1253. plt.ylim([0, len(graphy)])
  1254. plt.xlim([0, max(graphx) + 10])
  1255. # display numbers at each bar
  1256. for i, v in enumerate(graphx):
  1257. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  1258. # display grid for better visuals
  1259. plt.grid(True)
  1260. # plot the bar
  1261. labels = graphy
  1262. graphy = list(range(len(graphx)))
  1263. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  1264. plt.yticks(graphy, labels)
  1265. out = self.pcap_filepath.replace('.pcap', '_plot-' + degree_type + ' Degree of an IP' + file_ending)
  1266. # plt.tight_layout()
  1267. plt.savefig(out, dpi=500)
  1268. return out
  1269. def plot_big_conv_ext_stat(attr: str, title: str, xlabel: str, suffix: str):
  1270. """
  1271. Plots the desired statistc per connection as horizontal bar plot.
  1272. Included are 'half-open' connections, where only one packet is exchanged.
  1273. The given statistics table has to have at least the attributes 'ipAddressA', 'portA', 'ipAddressB',
  1274. 'portB' and the specified additional attribute.
  1275. Note: there may be cutoff/scaling problems within the plot if there is too little data.
  1276. :param attr: The desired statistic, named with respect to its attribute in the given statistics table
  1277. :param title: The title of the created plot
  1278. :param xlabel: The name of the x-axis of the created plot
  1279. :param suffix: The suffix of the created file, including file extension
  1280. :return: A filepath to the file containing the created plot
  1281. """
  1282. plt.gcf().clear()
  1283. result = self.stats_db.process_user_defined_query(
  1284. "SELECT ipAddressA, portA, ipAddressB, portB, %s FROM conv_statistics_extended" % attr)
  1285. if result:
  1286. graphy, graphx = [], []
  1287. # plot data in descending order
  1288. result = sorted(result, key=lambda r: r[4])
  1289. # compute plot data
  1290. for i, row in enumerate(result):
  1291. addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
  1292. # adjust the justification of strings to improve appearance
  1293. len_max = max(len(addr1), len(addr2))
  1294. addr1 = addr1.ljust(len_max)
  1295. addr2 = addr2.ljust(len_max)
  1296. # add plot data
  1297. graphy.append("%s\n%s" % (addr1, addr2))
  1298. graphx.append(row[4])
  1299. # have x axis and its label appear at the top (instead of bottom)
  1300. fig, ax = plt.subplots()
  1301. ax.xaxis.tick_top()
  1302. ax.xaxis.set_label_position("top")
  1303. # compute plot height in inches for scaling the plot
  1304. dist_mult_height = 0.55 # this value turned out to work well
  1305. plt_height = len(graphy) * dist_mult_height
  1306. # originally, a good title distance turned out to be 1.012 with a plot height of 52.8
  1307. title_distance = 1 + 0.012 * 52.8 / plt_height
  1308. plt.gcf().set_size_inches(plt.gcf().get_size_inches()[0], plt_height) # set plot height
  1309. plt.gcf().subplots_adjust(left=0.35)
  1310. # set additional plot parameters
  1311. plt.title(title, y=title_distance)
  1312. plt.xlabel(xlabel)
  1313. plt.ylabel('Connection')
  1314. width = 0.5
  1315. plt.grid(True)
  1316. plt.gca().margins(y=0) # removes the space between data and x-axis within the plot
  1317. # plot the above data, first use plain numbers as graphy to maintain sorting
  1318. plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=0.5, color='red', edgecolor='red')
  1319. # now change the y numbers to the respective address labels
  1320. plt.yticks(range(len(graphy)), graphy)
  1321. # save created figure
  1322. out = self.pcap_filepath.replace('.pcap', suffix)
  1323. plt.savefig(out, dpi=500)
  1324. return out
  1325. def plot_packets_per_connection(file_ending: str):
  1326. """
  1327. Plots the total number of exchanged packets per connection.
  1328. :param file_ending: The file extension for the output file containing the plot
  1329. :return: A filepath to the file containing the created plot
  1330. """
  1331. title = 'Number of exchanged packets per connection'
  1332. suffix = '_plot-PktCount per Connection Distribution' + file_ending
  1333. # plot data and return outpath
  1334. return plot_big_conv_ext_stat("pktsCount", title, "Number of packets", suffix)
  1335. def plot_avg_pkts_per_comm_interval(file_ending: str):
  1336. """
  1337. Plots the average number of exchanged packets per communication interval for every connection.
  1338. :param file_ending: The file extension for the output file containing the plot
  1339. :return: A filepath to the file containing the created plot
  1340. """
  1341. title = 'Average number of exchanged packets per communication interval'
  1342. suffix = '_plot-Avg PktCount Communication Interval Distribution' + file_ending
  1343. # plot data and return outpath
  1344. return plot_big_conv_ext_stat("avgIntervalPktCount", title, "Number of packets", suffix)
  1345. def plot_avg_time_between_comm_interval(file_ending: str):
  1346. """
  1347. Plots the average time between the communication intervals of every connection.
  1348. :param file_ending: The file extension for the output file containing the plot
  1349. :return: A filepath to the file containing the created plot
  1350. """
  1351. title = 'Average time between communication intervals in seconds'
  1352. suffix = '_plot-Avg Time Between Communication Intervals Distribution' + file_ending
  1353. # plot data and return outpath
  1354. return plot_big_conv_ext_stat("avgTimeBetweenIntervals", title, 'Average time between intervals', suffix)
  1355. def plot_avg_comm_interval_time(file_ending: str):
  1356. """
  1357. Plots the average duration of a communication interval of every connection.
  1358. :param file_ending: The file extension for the output file containing the plot
  1359. :return: A filepath to the file containing the created plot
  1360. """
  1361. title = 'Average duration of a communication interval in seconds'
  1362. suffix = '_plot-Avg Duration Communication Interval Distribution' + file_ending
  1363. # plot data and return outpath
  1364. return plot_big_conv_ext_stat("avgIntervalTime", title, 'Average interval time', suffix)
  1365. def plot_total_comm_duration(file_ending: str):
  1366. """
  1367. Plots the total communication duration of every connection.
  1368. :param file_ending: The file extension for the output file containing the plot
  1369. :return: A filepath to the file containing the created plot
  1370. """
  1371. title = 'Total communication duration in seconds'
  1372. suffix = '_plot-Total Communication Duration Distribution' + file_ending
  1373. # plot data and return outpath
  1374. return plot_big_conv_ext_stat("totalConversationDuration", title, 'Duration', suffix)
  1375. def plot_comm_histogram(attr: str, title: str, label: str, suffix: str):
  1376. """
  1377. Plots a histogram about the specified attribute for communications.
  1378. :param attr: The statistics attribute for this histogram
  1379. :param title: The title of the histogram
  1380. :param label: The xlabel of the histogram
  1381. :param suffix: The file suffix
  1382. :return: The path to the created plot
  1383. """
  1384. plt.gcf().clear()
  1385. result_raw = self.stats_db.process_user_defined_query(
  1386. "SELECT %s FROM conv_statistics_extended" % attr)
  1387. # return without plotting if no data available
  1388. if not result_raw:
  1389. return None
  1390. result = []
  1391. for entry in result_raw:
  1392. result.append(entry[0])
  1393. # if title would be cut off, set minimum width
  1394. plt_size = plt.gcf().get_size_inches()
  1395. min_width = len(title) * 0.12
  1396. if plt_size[0] < min_width:
  1397. plt.gcf().set_size_inches(min_width, plt_size[1]) # set plot size
  1398. # set additional plot parameters
  1399. plt.title(title)
  1400. plt.ylabel("Relative frequency of connections")
  1401. plt.xlabel(label)
  1402. plt.grid(True)
  1403. # create 11 bins
  1404. bins = []
  1405. max_val = max(result)
  1406. for i in range(0, 11):
  1407. bins.append(i * max_val / 10)
  1408. # set weights normalize histogram
  1409. weights = numpy.ones_like(result) / float(len(result))
  1410. # plot the above data, first use plain numbers as graphy to maintain sorting
  1411. plt.hist(result, bins=bins, weights=weights, color='red', edgecolor='red', align="mid", rwidth=0.5)
  1412. plt.xticks(bins)
  1413. # save created figure
  1414. out = self.pcap_filepath.replace('.pcap', suffix)
  1415. plt.savefig(out, dpi=500)
  1416. return out
  1417. def plot_histogram_degree(degree_type: str, title: str, label: str, suffix: str):
  1418. """
  1419. Plots a histogram about the specified type for the degree of an IP.
  1420. :param degree_type: The type of degree, i.e. inDegree, outDegree or overallDegree
  1421. :param title: The title of the histogram
  1422. :param label: The xlabel of the histogram
  1423. :param suffix: The file suffix
  1424. :return: The path to the created plot
  1425. """
  1426. plt.gcf().clear()
  1427. result_raw = self.get_filtered_degree(degree_type)
  1428. # return without plotting if no data available
  1429. if not result_raw:
  1430. return None
  1431. result = []
  1432. for entry in result_raw:
  1433. result.append(entry[1])
  1434. # set additional plot parameters
  1435. plt.title(title)
  1436. plt.ylabel("Relative frequency of IPs")
  1437. plt.xlabel(label)
  1438. plt.grid(True)
  1439. # create 11 bins
  1440. bins = []
  1441. max_val = max(result)
  1442. for i in range(0, 11):
  1443. bins.append(int(i * max_val / 10))
  1444. # set weights normalize histogram
  1445. weights = numpy.ones_like(result) / float(len(result))
  1446. # plot the above data, first use plain numbers as graphy to maintain sorting
  1447. plt.hist(result, bins=bins, weights=weights, color='red', edgecolor='red', align="mid", rwidth=0.5)
  1448. plt.xticks(bins)
  1449. # save created figure
  1450. out = self.pcap_filepath.replace('.pcap', suffix)
  1451. plt.savefig(out, dpi=500)
  1452. return out
  1453. ttl_out_path = plot_ttl('.' + file_format)
  1454. print(".", end="", flush=True)
  1455. mss_out_path = plot_mss('.' + file_format)
  1456. print(".", end="", flush=True)
  1457. win_out_path = plot_win('.' + file_format)
  1458. print(".", end="", flush=True)
  1459. protocol_out_path = plot_protocol('.' + file_format)
  1460. print(".", end="", flush=True)
  1461. plot_interval_pktCount = plot_interval_pkt_count('.' + file_format)
  1462. print(".", end="", flush=True)
  1463. if entropy:
  1464. plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + file_format)
  1465. print(".", end="", flush=True)
  1466. plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + file_format)
  1467. print(".", end="", flush=True)
  1468. plot_interval_ip_src_cum_ent = plot_interval_ip_cum_ent("src", '.' + file_format)
  1469. print(".", end="", flush=True)
  1470. plot_interval_ip_dst_cum_ent = plot_interval_ip_cum_ent("dst", '.' + file_format)
  1471. print(".", end="", flush=True)
  1472. plot_interval_new_ip = plot_interval_new_ip('.' + file_format)
  1473. print(".", end="", flush=True)
  1474. plot_interval_new_port = plot_interval_new_port('.' + file_format)
  1475. print(".", end="", flush=True)
  1476. plot_interval_new_ttl = plot_interval_new_ttl('.' + file_format)
  1477. print(".", end="", flush=True)
  1478. plot_interval_new_tos = plot_interval_new_tos('.' + file_format)
  1479. print(".", end="", flush=True)
  1480. plot_interval_new_win_size = plot_interval_new_win_size('.' + file_format)
  1481. print(".", end="", flush=True)
  1482. plot_interval_new_mss = plot_interval_new_mss('.' + file_format)
  1483. print(".", end="", flush=True)
  1484. plot_hist_indegree_out = plot_histogram_degree("inDegree", "Histogram - Ingoing degree per IP Address",
  1485. "Ingoing degree",
  1486. "_plot-Histogram Ingoing Degree per IP" + file_format)
  1487. print(".", end="", flush=True)
  1488. plot_hist_outdegree_out = plot_histogram_degree("outDegree", "Histogram - Outgoing degree per IP Address",
  1489. "Outgoing degree",
  1490. "_plot-Histogram Outgoing Degree per IP" + file_format)
  1491. print(".", end="", flush=True)
  1492. plot_hist_overalldegree_out = plot_histogram_degree("overallDegree",
  1493. "Histogram - Overall degree per IP Address",
  1494. "Overall degree",
  1495. "_plot-Histogram Overall Degree per IP" + file_format)
  1496. print(".", end="", flush=True)
  1497. plot_hist_pkts_per_connection_out = plot_comm_histogram("pktsCount",
  1498. "Histogram - Number of exchanged packets per connection",
  1499. "Number of packets",
  1500. "_plot-Histogram PktCount per Connection" + "." + file_format)
  1501. print(".", end="", flush=True)
  1502. plot_hist_avgpkts_per_commint_out = plot_comm_histogram("avgIntervalPktCount",
  1503. "Histogram - Average number of exchanged packets per communication interval",
  1504. "Average number of packets",
  1505. "_plot-Histogram Avg PktCount per Interval per Connection" + "." + file_format)
  1506. print(".", end="", flush=True)
  1507. plot_hist_avgtime_betw_commints_out = plot_comm_histogram("avgTimeBetweenIntervals",
  1508. "Histogram - Average time between communication intervals in seconds",
  1509. "Average time between intervals",
  1510. "_plot-Histogram Avg Time Between Intervals per Connection" + "." + file_format)
  1511. print(".", end="", flush=True)
  1512. plot_hist_avg_int_time_per_connection_out = plot_comm_histogram("avgIntervalTime",
  1513. "Histogram - Average duration of a communication interval in seconds",
  1514. "Average interval time",
  1515. "_plot-Histogram Avg Interval Time per Connection" + "." + file_format)
  1516. print(".", end="", flush=True)
  1517. plot_hist_total_comm_duration_out = plot_comm_histogram("totalConversationDuration",
  1518. "Histogram - Total communication duration in seconds",
  1519. "Duration",
  1520. "_plot-Histogram Communication Duration per Connection" + "." + file_format)
  1521. print(".", end="", flush=True)
  1522. if entropy:
  1523. plot_out_degree = plot_degree("out", '.' + file_format)
  1524. print(".", end="", flush=True)
  1525. plot_in_degree = plot_degree("in", '.' + file_format)
  1526. print(".", end="", flush=True)
  1527. plot_overall_degree = plot_degree("overall", '.' + file_format)
  1528. print(".", end="", flush=True)
  1529. plot_packets_per_connection_out = plot_packets_per_connection('.' + file_format)
  1530. print(".", end="", flush=True)
  1531. plot_avg_pkts_per_comm_interval_out = plot_avg_pkts_per_comm_interval('.' + file_format)
  1532. print(".", end="", flush=True)
  1533. plot_avg_time_between_comm_interval_out = plot_avg_time_between_comm_interval('.' + file_format)
  1534. print(".", end="", flush=True)
  1535. plot_avg_comm_interval_time_out = plot_avg_comm_interval_time("." + file_format)
  1536. print(".", end="", flush=True)
  1537. plot_total_comm_duration_out = plot_total_comm_duration("." + file_format)
  1538. print(".", end="", flush=True)
  1539. print(" done.")
  1540. # Time consuming plot
  1541. # port_out_path = plot_port('.' + format)
  1542. # Not drawable for too many IPs
  1543. # ip_src_out_path = plot_ip_src('.' + format)
  1544. # ip_dst_out_path = plot_ip_dst('.' + format)
  1545. print("Saved plots in the input PCAP directory.")
  1546. def stats_summary_post_attack(self, added_packets):
  1547. """
  1548. Prints a summary of relevant statistics after an attack is injected
  1549. :param added_packets: sum of packets added by attacks, gets updated if more than one attack
  1550. :return: None
  1551. """
  1552. total_packet_count = self.get_packet_count() + added_packets
  1553. added_packets_share = added_packets / total_packet_count * 100
  1554. timespan = self.get_capture_duration()
  1555. summary = [("Total packet count", total_packet_count, "packets"),
  1556. ("Added packet count", added_packets, "packets"),
  1557. ("Share of added packets", added_packets_share, "%"),
  1558. ("Capture duration", timespan, "seconds")]
  1559. print("\nPOST INJECTION STATISTICS SUMMARY --------------------------")
  1560. self.write_list(summary, print, "")
  1561. print("------------------------------------------------------------")
  1562. def stats_summary_new_db(self):
  1563. """
  1564. Prints a summary of relevant statistics when a new db is created
  1565. :return: None
  1566. """
  1567. self.file_info = self.stats_db.get_file_info()
  1568. print("\nNew database has been generated, printing statistics summary... ")
  1569. total_packet_count = self.get_packet_count()
  1570. pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
  1571. if pdu_count is None:
  1572. pdu_count = 0
  1573. pdu_share = pdu_count / total_packet_count * 100
  1574. last_pdu_timestamp = self.process_db_query(
  1575. "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")
  1576. timespan = self.get_capture_duration()
  1577. summary = [("Total packet count", total_packet_count, "packets"),
  1578. ("Recognized packets", total_packet_count - pdu_count, "packets"),
  1579. ("Unrecognized packets", pdu_count, "PDUs"),
  1580. ("% Recognized packets", 100 - pdu_share, "%"),
  1581. ("% Unrecognized packets", pdu_share, "%"),
  1582. ("Last unknown PDU", last_pdu_timestamp),
  1583. ("Capture duration", timespan, "seconds")]
  1584. print("\nPCAP FILE STATISTICS SUMMARY ------------------------------")
  1585. self.write_list(summary, print, "")
  1586. print("------------------------------------------------------------")