Statistics.py 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148
  1. # Aidmar
  2. from operator import itemgetter
  3. from math import sqrt, ceil, log
  4. import os
  5. import time
  6. import ID2TLib.libpcapreader as pr
  7. import matplotlib
  8. from numpy.random.mtrand import normal
  9. from scipy.linalg.misc import norm
  10. matplotlib.use('Agg')
  11. import matplotlib.pyplot as plt
  12. from ID2TLib.PcapFile import PcapFile
  13. from ID2TLib.StatsDatabase import StatsDatabase
  14. class Statistics:
  15. def __init__(self, pcap_file: PcapFile):
  16. """
  17. Creates a new Statistics object.
  18. :param pcap_file: A reference to the PcapFile object
  19. """
  20. # Fields
  21. self.pcap_filepath = pcap_file.pcap_file_path
  22. self.pcap_proc = None
  23. # Aidmar
  24. self.do_tests = False
  25. # Create folder for statistics database if required
  26. self.path_db = pcap_file.get_db_path()
  27. path_dir = os.path.dirname(self.path_db)
  28. if not os.path.isdir(path_dir):
  29. os.makedirs(path_dir)
  30. # Class instances
  31. self.stats_db = StatsDatabase(self.path_db)
  32. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool):
  33. """
  34. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  35. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  36. statistics are gathered directly from the existing database.
  37. :param flag_write_file: Indicates whether the statistics should be written addiotionally into a text file (True)
  38. or not (False)
  39. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  40. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  41. """
  42. # Load pcap and get loading time
  43. time_start = time.clock()
  44. # Inform user about recalculation of statistics and its reason
  45. if flag_recalculate_stats:
  46. print("Flag -r/--recalculate found. Recalculating statistics.")
  47. # Recalculate statistics if database not exists OR param -r/--recalculate was provided
  48. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats:
  49. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_tests)) # Aidmar - do_tests
  50. self.pcap_proc.collect_statistics()
  51. self.pcap_proc.write_to_database(self.path_db)
  52. outstring_datasource = "by PCAP file processor."
  53. else:
  54. outstring_datasource = "from statistics database."
  55. # Load statistics from database
  56. self.file_info = self.stats_db.get_file_info()
  57. time_end = time.clock()
  58. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  59. # Write statistics if param -e/--export provided
  60. if flag_write_file:
  61. self.write_statistics_to_file()
  62. # Print statistics if param -s/--statistics provided
  63. if flag_print_statistics:
  64. self.print_statistics()
  65. def get_file_information(self):
  66. """
  67. Returns a list of tuples, each containing a information of the file.
  68. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  69. """
  70. return [("Pcap file", self.pcap_filepath),
  71. ("#Packets", self.get_packet_count(), "packets"),
  72. ("Capture length", self.get_capture_duration(), "seconds"),
  73. ("Capture start", self.get_pcap_timestamp_start()),
  74. ("Capture end", self.get_pcap_timestamp_end())]
  75. def get_general_file_statistics(self):
  76. """
  77. Returns a list of tuples, each containing a file statistic.
  78. :return: a list of tuples, each consisting of (description, value, unit).
  79. """
  80. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  81. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  82. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  83. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  84. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  85. @staticmethod
  86. def write_list(desc_val_unit_list, func, line_ending="\n"):
  87. """
  88. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three values
  89. and applies the function func on this string.
  90. Before generating the string, it identifies text containing a float number, casts the string to a
  91. float and rounds the value to two decimal digits.
  92. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  93. :param func: The function to be applied to each generated string
  94. :param line_ending: The formatting string to be applied at the end of each string
  95. """
  96. for entry in desc_val_unit_list:
  97. # Convert text containing float into float
  98. (description, value) = entry[0:2]
  99. if isinstance(value, str) and "." in value:
  100. try:
  101. value = float(value)
  102. except ValueError:
  103. pass # do nothing -> value was not a float
  104. # round float
  105. if isinstance(value, float):
  106. value = round(value, 4)
  107. # write into file
  108. if len(entry) == 3:
  109. unit = entry[2]
  110. func(description + ":\t" + str(value) + " " + unit + line_ending)
  111. else:
  112. func(description + ":\t" + str(value) + line_ending)
  113. def print_statistics(self):
  114. """
  115. Prints the basic file statistics to the terminal.
  116. """
  117. print("\nPCAP FILE INFORMATION ------------------------------")
  118. Statistics.write_list(self.get_file_information(), print, "")
  119. print("\nGENERAL FILE STATISTICS ----------------------------")
  120. Statistics.write_list(self.get_general_file_statistics(), print, "")
  121. print("\n")
  122. #Aidmar
  123. def calculate_entropy(self, frequency:list, normalized:bool = False):
  124. entropy, normalizedEnt, n = 0, 0, 0
  125. sumFreq = sum(frequency)
  126. for i, x in enumerate(frequency):
  127. p_x = float(frequency[i] / sumFreq)
  128. if p_x > 0:
  129. n += 1
  130. entropy += - p_x * log(p_x, 2)
  131. if normalized:
  132. if log(n)>0:
  133. normalizedEnt = entropy/log(n, 2)
  134. return entropy, normalizedEnt
  135. else:
  136. return entropy
  137. # Aidmar
  138. def get_tests_statistics(self):
  139. """
  140. Writes the calculated basic defects tests statistics into a file.
  141. """
  142. # self.stats_db._process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
  143. def count_frequncy(valuesList):
  144. values, frequency = [] , []
  145. for x in valuesList:
  146. if x in values:
  147. frequency[values.index(x)] += 1
  148. else:
  149. values.append(x)
  150. frequency.append(1)
  151. return values, frequency
  152. ####### Payload Tests #######
  153. sumPayloadCount = self.stats_db._process_user_defined_query("SELECT sum(payloadCount) FROM interval_statistics")
  154. pktCount = self.stats_db._process_user_defined_query("SELECT packetCount FROM file_statistics")
  155. if sumPayloadCount and pktCount:
  156. payloadRatio=0
  157. if(pktCount[0][0]!=0):
  158. payloadRatio = float(sumPayloadCount[0][0] / pktCount[0][0] * 100)
  159. else:
  160. payloadRatio = -1
  161. ####### TCP checksum Tests #######
  162. incorrectChecksumCount = self.stats_db._process_user_defined_query("SELECT sum(incorrectTCPChecksumCount) FROM interval_statistics")
  163. correctChecksumCount = self.stats_db._process_user_defined_query("SELECT avg(correctTCPChecksumCount) FROM interval_statistics")
  164. if incorrectChecksumCount and correctChecksumCount:
  165. incorrectChecksumRatio=0
  166. if(incorrectChecksumCount[0][0] + correctChecksumCount[0][0])!=0:
  167. incorrectChecksumRatio = float(incorrectChecksumCount[0][0] / (incorrectChecksumCount[0][0] + correctChecksumCount[0][0] ) * 100)
  168. else:
  169. incorrectChecksumRatio = -1
  170. ####### IP Src Tests #######
  171. result = self.stats_db._process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
  172. data, srcFrequency, dstFrequency = [], [], []
  173. if result:
  174. for row in result:
  175. srcFrequency.append(row[1])
  176. dstFrequency.append(row[2])
  177. ipSrcEntropy, ipSrcNormEntropy = self.calculate_entropy(srcFrequency, True)
  178. ipDstEntropy, ipDstNormEntropy = self.calculate_entropy(dstFrequency, True)
  179. newIPCount = self.stats_db._process_user_defined_query("SELECT newIPCount FROM interval_statistics")
  180. ipNovelsPerInterval, ipNovelsPerIntervalFrequency = count_frequncy(newIPCount)
  181. ipNovelityDistEntropy = self.calculate_entropy(ipNovelsPerIntervalFrequency)
  182. # newIPCount = self.stats_db._process_user_defined_query("SELECT newIPCount FROM interval_statistics")
  183. # # Retrieve the last cumulative entropy which is the entropy of the all IPs
  184. # result = self.stats_db._process_user_defined_query("SELECT ipSrcCumEntropy FROM interval_statistics")
  185. # ipSrcEntropy = result[-1][0]
  186. # ipSrcCount = self.stats_db._process_user_defined_query(
  187. # "SELECT COUNT(ipAddress) FROM ip_statistics WHERE pktsSent > 0")
  188. # ipSrcNormEntropy = ipSrcEntropy / log(ipSrcCount[0][0],2)
  189. # result = self.stats_db._process_user_defined_query("SELECT ipDstCumEntropy FROM interval_statistics")
  190. # ipDstEntropy = result[-1][0]
  191. # ipDstCount = self.stats_db._process_user_defined_query(
  192. # "SELECT COUNT(ipAddress) FROM ip_statistics WHERE pktsReceived > 0")
  193. # ipDstNormEntropy = ipDstEntropy / log(ipDstCount[0][0],2)
  194. ####### Ports Tests #######
  195. port0Count = self.stats_db._process_user_defined_query("SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
  196. if not port0Count[0][0]:
  197. port0Count = 0
  198. else:
  199. port0Count = port0Count[0][0]
  200. reservedPortCount = self.stats_db._process_user_defined_query(
  201. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")# could be extended
  202. if not reservedPortCount[0][0]:
  203. reservedPortCount = 0
  204. else:
  205. reservedPortCount = reservedPortCount[0][0]
  206. ####### TTL Tests #######
  207. result = self.stats_db._process_user_defined_query("SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  208. data, frequency = [], []
  209. for row in result:
  210. frequency.append(row[1])
  211. ttlEntropy, ttlNormEntropy = self.calculate_entropy(frequency,True)
  212. newTTLCount = self.stats_db._process_user_defined_query("SELECT newTTLCount FROM interval_statistics")
  213. ttlNovelsPerInterval, ttlNovelsPerIntervalFrequency = count_frequncy(newTTLCount)
  214. ttlNovelityDistEntropy = self.calculate_entropy(ttlNovelsPerIntervalFrequency)
  215. ####### Window Size Tests #######
  216. result = self.stats_db._process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
  217. data, frequency = [], []
  218. for row in result:
  219. frequency.append(row[1])
  220. winEntropy, winNormEntropy = self.calculate_entropy(frequency, True)
  221. newWinSizeCount = self.stats_db._process_user_defined_query("SELECT newWinSizeCount FROM interval_statistics")
  222. winNovelsPerInterval, winNovelsPerIntervalFrequency = count_frequncy(newWinSizeCount)
  223. winNovelityDistEntropy = self.calculate_entropy(winNovelsPerIntervalFrequency)
  224. ####### ToS Tests #######
  225. result = self.stats_db._process_user_defined_query(
  226. "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
  227. data, frequency = [], []
  228. for row in result:
  229. frequency.append(row[1])
  230. tosEntropy, tosNormEntropy = self.calculate_entropy(frequency, True)
  231. newToSCount = self.stats_db._process_user_defined_query("SELECT newToSCount FROM interval_statistics")
  232. tosNovelsPerInterval, tosNovelsPerIntervalFrequency = count_frequncy(newToSCount)
  233. tosNovelityDistEntropy = self.calculate_entropy(tosNovelsPerIntervalFrequency)
  234. ####### MSS Tests #######
  235. result = self.stats_db._process_user_defined_query(
  236. "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  237. data, frequency = [], []
  238. for row in result:
  239. frequency.append(row[1])
  240. mssEntropy, mssNormEntropy = self.calculate_entropy(frequency, True)
  241. newMSSCount = self.stats_db._process_user_defined_query("SELECT newMSSCount FROM interval_statistics")
  242. mssNovelsPerInterval, mssNovelsPerIntervalFrequency = count_frequncy(newMSSCount)
  243. mssNovelityDistEntropy = self.calculate_entropy(mssNovelsPerIntervalFrequency)
  244. result = self.stats_db._process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
  245. # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
  246. if not result[0][0]:
  247. result = 0
  248. else:
  249. result = result[0][0]
  250. bigMSS = (result / sum(frequency)) * 100
  251. output = [("Payload ratio", payloadRatio, "%"),
  252. ("Incorrect TCP checksum ratio", incorrectChecksumRatio, "%"),
  253. ("# IP addresses", sum([x[0] for x in newIPCount]), ""),
  254. ("IP Src Entropy", ipSrcEntropy, ""),
  255. ("IP Src Normalized Entropy", ipSrcNormEntropy, ""),
  256. ("IP Dst Entropy", ipDstEntropy, ""),
  257. ("IP Dst Normalized Entropy", ipDstNormEntropy, ""),
  258. ("# TTL values", sum([x[0] for x in newTTLCount]), ""),
  259. ("TTL Distribution Entropy", ipNovelityDistEntropy, ""),
  260. ("TTL Entropy", ttlEntropy, ""),
  261. ("TTL Normalized Entropy", ttlNormEntropy, ""),
  262. ("TTL Distribution Entropy", ttlNovelityDistEntropy, ""),
  263. ("# WinSize values", sum([x[0] for x in newWinSizeCount]), ""),
  264. ("WinSize Entropy", winEntropy, ""),
  265. ("WinSize Normalized Entropy", winNormEntropy, ""),
  266. ("WinSize Distribution Entropy", winNovelityDistEntropy, ""),
  267. ("# ToS values", sum([x[0] for x in newToSCount]), ""),
  268. ("ToS Entropy", tosEntropy, ""),
  269. ("ToS Normalized Entropy", tosNormEntropy, ""),
  270. ("ToS Distribution Entropy", tosNovelityDistEntropy, ""),
  271. ("# MSS values", sum([x[0] for x in newMSSCount]), ""),
  272. ("MSS Entropy", mssEntropy, ""),
  273. ("MSS Normalized Entropy", mssNormEntropy, ""),
  274. ("MSS Distribution Entropy", mssNovelityDistEntropy, ""),
  275. ("======================","","")]
  276. # Reasoning the statistics values
  277. if payloadRatio > 80:
  278. output.append(("WARNING: Too high payload ratio", payloadRatio, "%."))
  279. if payloadRatio < 30:
  280. output.append(("WARNING: Too low payload ratio", payloadRatio, "% (Injecting attacks that are carried out in the packet payloads is not recommmanded)."))
  281. if incorrectChecksumRatio > 5:
  282. output.append(("WARNING: High incorrect TCP checksum ratio",incorrectChecksumRatio,"%."))
  283. if ipSrcNormEntropy > 0.65:
  284. output.append(("WARNING: High IP source normalized entropy",ipSrcNormEntropy,"."))
  285. if ipSrcNormEntropy < 0.2:
  286. output.append(("WARNING: Low IP source normalized entropy", ipSrcNormEntropy, "."))
  287. if ipDstNormEntropy > 0.65:
  288. output.append(("WARNING: High IP destination normalized entropy", ipDstNormEntropy, "."))
  289. if ipDstNormEntropy < 0.2:
  290. output.append(("WARNING: Low IP destination normalized entropy", ipDstNormEntropy, "."))
  291. if ttlNormEntropy > 0.65:
  292. output.append(("WARNING: High TTL normalized entropy", ttlNormEntropy, "."))
  293. if ttlNormEntropy < 0.2:
  294. output.append(("WARNING: Low TTL normalized entropy", ttlNormEntropy, "."))
  295. if ttlNovelityDistEntropy < 1:
  296. output.append(("WARNING: Too low TTL novelity distribution entropy", ttlNovelityDistEntropy,
  297. "(The distribution of the novel TTL values is suspicious)."))
  298. if winNormEntropy > 0.6:
  299. output.append(("WARNING: High Window Size normalized entropy", winNormEntropy, "."))
  300. if winNormEntropy < 0.1:
  301. output.append(("WARNING: Low Window Size normalized entropy", winNormEntropy, "."))
  302. if winNovelityDistEntropy < 4:
  303. output.append(("WARNING: Low Window Size novelity distribution entropy", winNovelityDistEntropy,
  304. "(The distribution of the novel Window Size values is suspicious)."))
  305. if tosNormEntropy > 0.4:
  306. output.append(("WARNING: High ToS normalized entropy", tosNormEntropy, "."))
  307. if tosNormEntropy < 0.1:
  308. output.append(("WARNING: Low ToS normalized entropy", tosNormEntropy, "."))
  309. if tosNovelityDistEntropy < 0.5:
  310. output.append(("WARNING: Low ToS novelity distribution entropy", tosNovelityDistEntropy,
  311. "(The distribution of the novel ToS values is suspicious)."))
  312. if mssNormEntropy > 0.4:
  313. output.append(("WARNING: High MSS normalized entropy", mssNormEntropy, "."))
  314. if mssNormEntropy < 0.1:
  315. output.append(("WARNING: Low MSS normalized entropy", mssNormEntropy, "."))
  316. if mssNovelityDistEntropy < 0.5:
  317. output.append(("WARNING: Low MSS novelity distribution entropy", mssNovelityDistEntropy,
  318. "(The distribution of the novel MSS values is suspicious)."))
  319. if bigMSS > 50:
  320. output.append(("WARNING: High ratio of MSS > 1460", bigMSS, "% (High fragmentation rate in Ethernet)."))
  321. if port0Count > 0:
  322. output.append(("WARNING: Port number 0 is used in ",port0Count,"packets (awkward-looking port)."))
  323. if reservedPortCount > 0:
  324. output.append(("WARNING: Reserved port numbers are used in ",reservedPortCount,"packets (uncommonly-used ports)."))
  325. return output
  326. def write_statistics_to_file(self):
  327. """
  328. Writes the calculated basic statistics into a file.
  329. """
  330. def _write_header(title: str):
  331. """
  332. Writes the section header into the open file.
  333. :param title: The section title
  334. """
  335. target.write("====================== \n")
  336. target.write(title + " \n")
  337. target.write("====================== \n")
  338. target = open(self.pcap_filepath + ".stat", 'w')
  339. target.truncate()
  340. _write_header("PCAP file information")
  341. Statistics.write_list(self.get_file_information(), target.write)
  342. _write_header("General statistics")
  343. Statistics.write_list(self.get_general_file_statistics(), target.write)
  344. _write_header("Tests statistics")
  345. Statistics.write_list(self.get_tests_statistics(), target.write)
  346. target.close()
  347. def get_capture_duration(self):
  348. """
  349. :return: The duration of the capture in seconds
  350. """
  351. return self.file_info['captureDuration']
  352. def get_pcap_timestamp_start(self):
  353. """
  354. :return: The timestamp of the first packet in the PCAP file
  355. """
  356. return self.file_info['timestampFirstPacket']
  357. def get_pcap_timestamp_end(self):
  358. """
  359. :return: The timestamp of the last packet in the PCAP file
  360. """
  361. return self.file_info['timestampLastPacket']
  362. def get_pps_sent(self, ip_address: str):
  363. """
  364. Calculates the sent packets per seconds for a given IP address.
  365. :param ip_address: The IP address whose packets per second should be calculated
  366. :return: The sent packets per seconds for the given IP address
  367. """
  368. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  369. (ip_address,))
  370. capture_duration = float(self.get_capture_duration())
  371. return int(float(packets_sent) / capture_duration)
  372. def get_pps_received(self, ip_address: str):
  373. """
  374. Calculate the packets per second received for a given IP address.
  375. :param ip_address: The IP address used for the calculation
  376. :return: The number of packets per second received
  377. """
  378. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  379. False,
  380. (ip_address,))
  381. capture_duration = float(self.get_capture_duration())
  382. return int(float(packets_received) / capture_duration)
  383. def get_packet_count(self):
  384. """
  385. :return: The number of packets in the loaded PCAP file
  386. """
  387. return self.file_info['packetCount']
  388. def get_most_used_ip_address(self):
  389. """
  390. :return: The IP address/addresses with the highest sum of packets sent and received
  391. """
  392. return self.process_db_query("most_used(ipAddress)")
  393. def get_ttl_distribution(self, ipAddress: str):
  394. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ipAddress + '"')
  395. result_dict = {key: value for (key, value) in result}
  396. return result_dict
  397. # Aidmar
  398. def get_mss_distribution(self, ipAddress: str):
  399. result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ipAddress + '"')
  400. result_dict = {key: value for (key, value) in result}
  401. return result_dict
  402. # Aidmar
  403. def get_win_distribution(self, ipAddress: str):
  404. result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ipAddress + '"')
  405. result_dict = {key: value for (key, value) in result}
  406. return result_dict
  407. # Aidmar
  408. def get_tos_distribution(self, ipAddress: str):
  409. result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ipAddress + '"')
  410. result_dict = {key: value for (key, value) in result}
  411. return result_dict
  412. def get_random_ip_address(self, count: int = 1):
  413. """
  414. :param count: The number of IP addreses to return
  415. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of randomly
  416. chosen IP addresses
  417. """
  418. if count == 1:
  419. return self.process_db_query("random(all(ipAddress))")
  420. else:
  421. ip_address_list = []
  422. for i in range(0, count):
  423. ip_address_list.append(self.process_db_query("random(all(ipAddress))"))
  424. return ip_address_list
  425. def get_mac_address(self, ipAddress: str):
  426. """
  427. :return: The MAC address used in the dataset for the given IP address.
  428. """
  429. return self.process_db_query('macAddress(ipAddress=' + ipAddress + ")")
  430. # Aidmar
  431. def get_most_used_mss(self, ipAddress: str):
  432. """
  433. :param ipAddress: The IP address whose used MSS should be determined
  434. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  435. then None is returned
  436. """
  437. mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ipAddress + '" ORDER BY mssCount DESC LIMIT 1')
  438. if isinstance(mss_value, int):
  439. return mss_value
  440. else:
  441. return None
  442. def get_statistics_database(self):
  443. """
  444. :return: A reference to the statistics database object
  445. """
  446. return self.stats_db
  447. def process_db_query(self, query_string_in: str, print_results: bool = False):
  448. """
  449. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  450. query.
  451. :param query_string_in: The query to be processed
  452. :param print_results: Indicates whether the results should be printed to terminal
  453. :return: The result of the query
  454. """
  455. return self.stats_db.process_db_query(query_string_in, print_results)
  456. def is_query(self, value: str):
  457. """
  458. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  459. :param value: The string to be checked
  460. :return: True if the string is recognized as a query, otherwise False.
  461. """
  462. if not isinstance(value, str):
  463. return False
  464. else:
  465. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  466. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  467. # Aidmar
  468. def calculate_standard_deviation(self, lst):
  469. """Calculates the standard deviation for a list of numbers."""
  470. num_items = len(lst)
  471. mean = sum(lst) / num_items
  472. differences = [x - mean for x in lst]
  473. sq_differences = [d ** 2 for d in differences]
  474. ssd = sum(sq_differences)
  475. variance = ssd / num_items
  476. sd = sqrt(variance)
  477. #print('The mean of {} is {}.'.format(lst, mean))
  478. #print('The differences are {}.'.format(differences))
  479. #print('The sum of squared differences is {}.'.format(ssd))
  480. #print('The variance is {}.'.format(variance))
  481. #print('The standard deviation is {}.'.format(sd))
  482. #print('--------------------------')
  483. return sd
  484. def plot_statistics(self, format: str = 'pdf'): #'png'):
  485. """
  486. Plots the statistics associated with the dataset prior attack injection.
  487. :param format: The format to be used to save the statistics diagrams.
  488. """
  489. def plot_ttl(file_ending: str):
  490. plt.gcf().clear()
  491. result = self.stats_db._process_user_defined_query(
  492. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  493. graphx, graphy = [], []
  494. for row in result:
  495. graphx.append(row[0])
  496. graphy.append(row[1])
  497. plt.autoscale(enable=True, axis='both')
  498. plt.title("TTL Distribution")
  499. plt.xlabel('TTL Value')
  500. plt.ylabel('Number of Packets')
  501. width = 0.1
  502. plt.xlim([0, max(graphx)])
  503. plt.grid(True)
  504. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  505. out = self.pcap_filepath.replace('.pcap', '_plot-ttl' + file_ending)
  506. plt.savefig(out,dpi=500)
  507. return out
  508. # Aidmar
  509. def plot_mss(file_ending: str):
  510. plt.gcf().clear()
  511. result = self.stats_db._process_user_defined_query(
  512. "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  513. if(result):
  514. graphx, graphy = [], []
  515. for row in result:
  516. graphx.append(row[0])
  517. graphy.append(row[1])
  518. plt.autoscale(enable=True, axis='both')
  519. plt.title("MSS Distribution")
  520. plt.xlabel('MSS Value')
  521. plt.ylabel('Number of Packets')
  522. width = 0.1
  523. plt.xlim([0, max(graphx)])
  524. plt.grid(True)
  525. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  526. out = self.pcap_filepath.replace('.pcap', '_plot-mss' + file_ending)
  527. plt.savefig(out,dpi=500)
  528. return out
  529. else:
  530. print("Error plot MSS: No MSS values found!")
  531. # Aidmar
  532. def plot_win(file_ending: str):
  533. plt.gcf().clear()
  534. result = self.stats_db._process_user_defined_query(
  535. "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
  536. if (result):
  537. graphx, graphy = [], []
  538. for row in result:
  539. graphx.append(row[0])
  540. graphy.append(row[1])
  541. plt.autoscale(enable=True, axis='both')
  542. plt.title("Window Size Distribution")
  543. plt.xlabel('Window Size')
  544. plt.ylabel('Number of Packets')
  545. width = 0.1
  546. plt.xlim([0, max(graphx)])
  547. plt.grid(True)
  548. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  549. out = self.pcap_filepath.replace('.pcap', '_plot-win' + file_ending)
  550. plt.savefig(out,dpi=500)
  551. return out
  552. else:
  553. print("Error plot WinSize: No WinSize values found!")
  554. # Aidmar
  555. def plot_protocol(file_ending: str):
  556. plt.gcf().clear()
  557. result = self.stats_db._process_user_defined_query(
  558. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  559. if (result):
  560. graphx, graphy = [], []
  561. for row in result:
  562. graphx.append(row[0])
  563. graphy.append(row[1])
  564. plt.autoscale(enable=True, axis='both')
  565. plt.title("Protocols Distribution")
  566. plt.xlabel('Protocols')
  567. plt.ylabel('Number of Packets')
  568. width = 0.1
  569. plt.xlim([0, len(graphx)])
  570. plt.grid(True)
  571. # Protocols' names on x-axis
  572. x = range(0,len(graphx))
  573. my_xticks = graphx
  574. plt.xticks(x, my_xticks)
  575. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  576. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  577. plt.savefig(out,dpi=500)
  578. return out
  579. else:
  580. print("Error plot protocol: No protocol values found!")
  581. # Aidmar
  582. def plot_port(file_ending: str):
  583. plt.gcf().clear()
  584. result = self.stats_db._process_user_defined_query(
  585. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  586. graphx, graphy = [], []
  587. for row in result:
  588. graphx.append(row[0])
  589. graphy.append(row[1])
  590. plt.autoscale(enable=True, axis='both')
  591. plt.title("Ports Distribution")
  592. plt.xlabel('Ports Numbers')
  593. plt.ylabel('Number of Packets')
  594. width = 0.1
  595. plt.xlim([0, max(graphx)])
  596. plt.grid(True)
  597. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  598. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  599. plt.savefig(out,dpi=500)
  600. return out
  601. # Aidmar - This distribution is not drawable for big datasets
  602. def plot_ip_src(file_ending: str):
  603. plt.gcf().clear()
  604. result = self.stats_db._process_user_defined_query(
  605. "SELECT ipAddress, pktsSent FROM ip_statistics")
  606. graphx, graphy = [], []
  607. for row in result:
  608. graphx.append(row[0])
  609. graphy.append(row[1])
  610. plt.autoscale(enable=True, axis='both')
  611. plt.title("Source IP Distribution")
  612. plt.xlabel('Source IP')
  613. plt.ylabel('Number of Packets')
  614. width = 0.1
  615. plt.xlim([0, len(graphx)])
  616. plt.grid(True)
  617. # IPs on x-axis
  618. x = range(0, len(graphx))
  619. my_xticks = graphx
  620. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  621. plt.tight_layout()
  622. # limit the number of xticks
  623. plt.locator_params(axis='x', nbins=20)
  624. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  625. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  626. plt.savefig(out, dpi=500)
  627. return out
  628. # Aidmar - This distribution is not drawable for big datasets
  629. def plot_ip_dst(file_ending: str):
  630. plt.gcf().clear()
  631. result = self.stats_db._process_user_defined_query(
  632. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  633. graphx, graphy = [], []
  634. for row in result:
  635. graphx.append(row[0])
  636. graphy.append(row[1])
  637. plt.autoscale(enable=True, axis='both')
  638. plt.title("Destination IP Distribution")
  639. plt.xlabel('Destination IP')
  640. plt.ylabel('Number of Packets')
  641. width = 0.1
  642. plt.xlim([0, len(graphx)])
  643. plt.grid(True)
  644. # IPs on x-axis
  645. x = range(0, len(graphx))
  646. my_xticks = graphx
  647. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  648. plt.tight_layout()
  649. # limit the number of xticks
  650. plt.locator_params(axis='x', nbins=20)
  651. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  652. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  653. plt.savefig(out, dpi=500)
  654. return out
  655. # Aidmar
  656. def plot_interval_pktCount(file_ending: str):
  657. plt.gcf().clear()
  658. result = self.stats_db._process_user_defined_query(
  659. "SELECT lastPktTimestamp, pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  660. graphx, graphy = [], []
  661. for row in result:
  662. graphx.append(row[0])
  663. graphy.append(row[1])
  664. plt.autoscale(enable=True, axis='both')
  665. plt.title("Packet Rate")
  666. plt.xlabel('Timestamp')
  667. plt.ylabel('Number of Packets')
  668. width = 0.5
  669. plt.xlim([0, len(graphx)])
  670. plt.grid(True)
  671. # timestamp on x-axis
  672. x = range(0, len(graphx))
  673. my_xticks = graphx
  674. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  675. plt.tight_layout()
  676. # limit the number of xticks
  677. plt.locator_params(axis='x', nbins=20)
  678. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  679. out = self.pcap_filepath.replace('.pcap', '_plot-interval-pkt-count' + file_ending)
  680. plt.savefig(out, dpi=500)
  681. return out
  682. # Aidmar
  683. def plot_interval_ip_src_ent(file_ending: str):
  684. plt.gcf().clear()
  685. result = self.stats_db._process_user_defined_query(
  686. "SELECT lastPktTimestamp, ipSrcEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  687. graphx, graphy = [], []
  688. for row in result:
  689. graphx.append(row[0])
  690. graphy.append(row[1])
  691. plt.autoscale(enable=True, axis='both')
  692. plt.title("Source IP Entropy")
  693. plt.xlabel('Timestamp')
  694. plt.ylabel('Entropy')
  695. width = 0.5
  696. plt.xlim([0, len(graphx)])
  697. plt.grid(True)
  698. # timestamp on x-axis
  699. x = range(0, len(graphx))
  700. my_xticks = graphx
  701. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  702. plt.tight_layout()
  703. # limit the number of xticks
  704. plt.locator_params(axis='x', nbins=20)
  705. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  706. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-ent' + file_ending)
  707. plt.savefig(out, dpi=500)
  708. return out
  709. # Aidmar
  710. def plot_interval_ip_dst_ent(file_ending: str):
  711. plt.gcf().clear()
  712. result = self.stats_db._process_user_defined_query(
  713. "SELECT lastPktTimestamp, ipDstEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  714. graphx, graphy = [], []
  715. for row in result:
  716. graphx.append(row[0])
  717. graphy.append(row[1])
  718. plt.autoscale(enable=True, axis='both')
  719. plt.title("Destination IP Entropy")
  720. plt.xlabel('Timestamp')
  721. plt.ylabel('Entropy')
  722. width = 0.5
  723. plt.xlim([0, len(graphx)])
  724. plt.grid(True)
  725. # timestamp on x-axis
  726. x = range(0, len(graphx))
  727. my_xticks = graphx
  728. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  729. plt.tight_layout()
  730. # limit the number of xticks
  731. plt.locator_params(axis='x', nbins=20)
  732. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  733. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-ent' + file_ending)
  734. plt.savefig(out, dpi=500)
  735. return out
  736. # Aidmar
  737. def plot_interval_ip_dst_cum_ent(file_ending: str):
  738. plt.gcf().clear()
  739. result = self.stats_db._process_user_defined_query(
  740. "SELECT lastPktTimestamp, ipDstCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  741. graphx, graphy = [], []
  742. for row in result:
  743. graphx.append(row[0])
  744. graphy.append(row[1])
  745. plt.autoscale(enable=True, axis='both')
  746. plt.title("Destination IP Cumulative Entropy")
  747. plt.xlabel('Timestamp')
  748. plt.ylabel('Entropy')
  749. plt.xlim([0, len(graphx)])
  750. plt.grid(True)
  751. # timestamp on x-axis
  752. x = range(0, len(graphx))
  753. my_xticks = graphx
  754. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  755. plt.tight_layout()
  756. # limit the number of xticks
  757. plt.locator_params(axis='x', nbins=20)
  758. plt.plot(x, graphy, 'r')
  759. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-cum-ent' + file_ending)
  760. plt.savefig(out, dpi=500)
  761. return out
  762. # Aidmar
  763. def plot_interval_ip_src_cum_ent(file_ending: str):
  764. plt.gcf().clear()
  765. result = self.stats_db._process_user_defined_query(
  766. "SELECT lastPktTimestamp, ipSrcCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  767. graphx, graphy = [], []
  768. for row in result:
  769. graphx.append(row[0])
  770. graphy.append(row[1])
  771. plt.autoscale(enable=True, axis='both')
  772. plt.title("Source IP Cumulative Entropy")
  773. plt.xlabel('Timestamp')
  774. plt.ylabel('Entropy')
  775. plt.xlim([0, len(graphx)])
  776. plt.grid(True)
  777. # timestamp on x-axis
  778. x = range(0, len(graphx))
  779. my_xticks = graphx
  780. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  781. plt.tight_layout()
  782. # limit the number of xticks
  783. plt.locator_params(axis='x',nbins=20)
  784. plt.plot(x, graphy, 'r')
  785. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-cum-ent' + file_ending)
  786. plt.savefig(out, dpi=500)
  787. return out
  788. # Aidmar
  789. def plot_interval_new_ip(file_ending: str):
  790. plt.gcf().clear()
  791. result = self.stats_db._process_user_defined_query(
  792. "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
  793. graphx, graphy = [], []
  794. for row in result:
  795. graphx.append(row[0])
  796. graphy.append(row[1])
  797. plt.autoscale(enable=True, axis='both')
  798. plt.title("IP Novelity Distribution")
  799. plt.xlabel('Timestamp')
  800. plt.ylabel('Novel values count')
  801. plt.xlim([0, len(graphx)])
  802. plt.grid(True)
  803. width = 0.5
  804. # timestamp on x-axis
  805. x = range(0, len(graphx))
  806. my_xticks = graphx
  807. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  808. plt.tight_layout()
  809. # limit the number of xticks
  810. plt.locator_params(axis='x', nbins=20)
  811. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  812. out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-ip-dist' + file_ending)
  813. plt.savefig(out, dpi=500)
  814. #print("IP Standard Deviation:")
  815. #self.calculate_standard_deviation(graphy)
  816. return out
  817. # Aidmar
  818. def plot_interval_new_ttl(file_ending: str):
  819. plt.gcf().clear()
  820. result = self.stats_db._process_user_defined_query(
  821. "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
  822. if(result):
  823. graphx, graphy = [], []
  824. for row in result:
  825. graphx.append(row[0])
  826. graphy.append(row[1])
  827. plt.autoscale(enable=True, axis='both')
  828. plt.title("TTL Novelity Distribution")
  829. plt.xlabel('Timestamp')
  830. plt.ylabel('Novel values count')
  831. plt.xlim([0, len(graphx)])
  832. plt.grid(True)
  833. width = 0.5
  834. # timestamp on x-axis
  835. x = range(0, len(graphx))
  836. my_xticks = graphx
  837. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  838. plt.tight_layout()
  839. # limit the number of xticks
  840. plt.locator_params(axis='x', nbins=20)
  841. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  842. out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-ttl-dist' + file_ending)
  843. plt.savefig(out, dpi=500)
  844. #print("TTL Standard Deviation:")
  845. #self.calculate_standard_deviation(graphy)
  846. return out
  847. else:
  848. print("Error plot TTL: No TTL values found!")
  849. # Aidmar
  850. def plot_interval_new_tos(file_ending: str):
  851. plt.gcf().clear()
  852. result = self.stats_db._process_user_defined_query(
  853. "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  854. graphx, graphy = [], []
  855. for row in result:
  856. graphx.append(row[0])
  857. graphy.append(row[1])
  858. plt.autoscale(enable=True, axis='both')
  859. plt.title("ToS Novelity Distribution")
  860. plt.xlabel('Timestamp')
  861. plt.ylabel('Novel values count')
  862. plt.xlim([0, len(graphx)])
  863. plt.grid(True)
  864. width = 0.5
  865. # timestamp on x-axis
  866. x = range(0, len(graphx))
  867. my_xticks = graphx
  868. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  869. plt.tight_layout()
  870. # limit the number of xticks
  871. plt.locator_params(axis='x', nbins=20)
  872. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  873. out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-tos-dist' + file_ending)
  874. plt.savefig(out, dpi=500)
  875. #print("ToS Standard Deviation:")
  876. #self.calculate_standard_deviation(graphy)
  877. return out
  878. # Aidmar
  879. def plot_interval_new_win_size(file_ending: str):
  880. plt.gcf().clear()
  881. result = self.stats_db._process_user_defined_query(
  882. "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
  883. if(result):
  884. graphx, graphy = [], []
  885. for row in result:
  886. graphx.append(row[0])
  887. graphy.append(row[1])
  888. plt.autoscale(enable=True, axis='both')
  889. plt.title("Window Size Novelity Distribution")
  890. plt.xlabel('Timestamp')
  891. plt.ylabel('Novel values count')
  892. plt.xlim([0, len(graphx)])
  893. plt.grid(True)
  894. width = 0.5
  895. # timestamp on x-axis
  896. x = range(0, len(graphx))
  897. my_xticks = graphx
  898. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  899. plt.tight_layout()
  900. # limit the number of xticks
  901. plt.locator_params(axis='x', nbins=20)
  902. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  903. out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-win-size-dist' + file_ending)
  904. plt.savefig(out, dpi=500)
  905. # Calculate Standart Deviation
  906. #print("Window Size Standard Deviation:")
  907. #self.calculate_standard_deviation(graphy)
  908. return out
  909. else:
  910. print("Error plot new values WinSize: No WinSize values found!")
  911. # Aidmar
  912. def plot_interval_new_mss(file_ending: str):
  913. plt.gcf().clear()
  914. result = self.stats_db._process_user_defined_query(
  915. "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  916. if(result):
  917. graphx, graphy = [], []
  918. for row in result:
  919. graphx.append(row[0])
  920. graphy.append(row[1])
  921. plt.autoscale(enable=True, axis='both')
  922. plt.title("MSS Novelity Distribution")
  923. plt.xlabel('Timestamp')
  924. plt.ylabel('Novel values count')
  925. plt.xlim([0, len(graphx)])
  926. plt.grid(True)
  927. width = 0.5
  928. # timestamp on x-axis
  929. x = range(0, len(graphx))
  930. my_xticks = graphx
  931. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  932. plt.tight_layout()
  933. # limit the number of xticks
  934. plt.locator_params(axis='x', nbins=20)
  935. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  936. out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-mss-dist' + file_ending)
  937. plt.savefig(out, dpi=500)
  938. # Calculate Standart Deviation
  939. #print("MSS Standard Deviation:")
  940. #self.calculate_standard_deviation(graphy)
  941. return out
  942. else:
  943. print("Error plot new values MSS: No MSS values found!")
  944. ttl_out_path = plot_ttl('.' + format)
  945. mss_out_path = plot_mss('.' + format)
  946. win_out_path = plot_win('.' + format)
  947. protocol_out_path = plot_protocol('.' + format)
  948. port_out_path = plot_port('.' + format)
  949. #ip_src_out_path = plot_ip_src('.' + format)
  950. #ip_dst_out_path = plot_ip_dst('.' + format)
  951. plot_interval_pktCount = plot_interval_pktCount('.' + format)
  952. plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + format)
  953. plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + format)
  954. plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + format)
  955. plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + format)
  956. plot_interval_new_ip = plot_interval_new_ip('.' + format)
  957. plot_interval_new_ttl = plot_interval_new_ttl('.' + format)
  958. plot_interval_new_tos = plot_interval_new_tos('.' + format)
  959. plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
  960. plot_interval_new_mss = plot_interval_new_mss('.' + format)
  961. #print("Saved distributions plots at: %s, %s, %s, %s, %s, %s, %s, %s %s" %(ttl_out_path,mss_out_path, win_out_path,
  962. #protocol_out_path, port_out_path,ip_src_out_path,ip_dst_out_path, plot_interval_pktCount))
  963. # Aidmar
  964. def calculate_complement_packet_rates(self, pps):
  965. """
  966. Calculates the complement packet rates of the background traffic packet rates for each interval.
  967. Then normalize it to maximum boundary, which is the input parameter pps
  968. :return: normalized packet rates for each time interval.
  969. """
  970. result = self.process_db_query(
  971. "SELECT lastPktTimestamp,pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  972. # print(result)
  973. bg_interval_pps = []
  974. complement_interval_pps = []
  975. intervalsSum = 0
  976. if result:
  977. # Get the interval in seconds
  978. for i, row in enumerate(result):
  979. if i < len(result) - 1:
  980. intervalsSum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
  981. interval = intervalsSum / (len(result) - 1)
  982. # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
  983. for row in result:
  984. bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
  985. # Find max PPS
  986. maxPPS = max(bg_interval_pps, key=itemgetter(1))[1]
  987. for row in bg_interval_pps:
  988. complement_interval_pps.append((row[0], int(pps * (maxPPS - row[1]) / maxPPS)))
  989. return complement_interval_pps