Statistics.py 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379
  1. from operator import itemgetter
  2. from math import sqrt, ceil, log
  3. import os
  4. import time
  5. import ID2TLib.libpcapreader as pr
  6. import matplotlib
  7. matplotlib.use('Agg')
  8. import matplotlib.pyplot as plt
  9. from ID2TLib.PcapFile import PcapFile
  10. from ID2TLib.StatsDatabase import StatsDatabase
  11. from ID2TLib.IPv4 import IPAddress
  12. class Statistics:
  13. def __init__(self, pcap_file: PcapFile):
  14. """
  15. Creates a new Statistics object.
  16. :param pcap_file: A reference to the PcapFile object
  17. """
  18. # Fields
  19. self.pcap_filepath = pcap_file.pcap_file_path
  20. self.pcap_proc = None
  21. self.do_extra_tests = False
  22. # Create folder for statistics database if required
  23. self.path_db = pcap_file.get_db_path()
  24. path_dir = os.path.dirname(self.path_db)
  25. if not os.path.isdir(path_dir):
  26. os.makedirs(path_dir)
  27. # Class instances
  28. self.stats_db = StatsDatabase(self.path_db)
  29. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool):
  30. """
  31. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  32. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  33. statistics are gathered directly from the existing database.
  34. :param flag_write_file: Indicates whether the statistics should be written addiotionally into a text file (True)
  35. or not (False)
  36. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  37. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  38. """
  39. # Load pcap and get loading time
  40. time_start = time.clock()
  41. # Inform user about recalculation of statistics and its reason
  42. if flag_recalculate_stats:
  43. print("Flag -r/--recalculate found. Recalculating statistics.")
  44. # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
  45. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats:
  46. self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests))
  47. self.pcap_proc.collect_statistics()
  48. self.pcap_proc.write_to_database(self.path_db)
  49. outstring_datasource = "by PCAP file processor."
  50. else:
  51. outstring_datasource = "from statistics database."
  52. # Load statistics from database
  53. self.file_info = self.stats_db.get_file_info()
  54. time_end = time.clock()
  55. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  56. # Write statistics if param -e/--export provided
  57. if flag_write_file:
  58. self.write_statistics_to_file()
  59. # Print statistics if param -s/--statistics provided
  60. if flag_print_statistics:
  61. self.print_statistics()
  62. def get_file_information(self):
  63. """
  64. Returns a list of tuples, each containing a information of the file.
  65. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  66. """
  67. return [("Pcap file", self.pcap_filepath),
  68. ("Packets", self.get_packet_count(), "packets"),
  69. ("Capture length", self.get_capture_duration(), "seconds"),
  70. ("Capture start", self.get_pcap_timestamp_start()),
  71. ("Capture end", self.get_pcap_timestamp_end())]
  72. def get_general_file_statistics(self):
  73. """
  74. Returns a list of tuples, each containing a file statistic.
  75. :return: a list of tuples, each consisting of (description, value, unit).
  76. """
  77. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  78. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  79. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  80. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  81. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  82. @staticmethod
  83. def write_list(desc_val_unit_list, func, line_ending="\n"):
  84. """
  85. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three values
  86. and applies the function func on this string.
  87. Before generating the string, it identifies text containing a float number, casts the string to a
  88. float and rounds the value to two decimal digits.
  89. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  90. :param func: The function to be applied to each generated string
  91. :param line_ending: The formatting string to be applied at the end of each string
  92. """
  93. for entry in desc_val_unit_list:
  94. # Convert text containing float into float
  95. (description, value) = entry[0:2]
  96. if isinstance(value, str) and "." in value:
  97. try:
  98. value = float(value)
  99. except ValueError:
  100. pass # do nothing -> value was not a float
  101. # round float
  102. if isinstance(value, float):
  103. value = round(value, 4)
  104. # write into file
  105. if len(entry) == 3:
  106. unit = entry[2]
  107. func(description + ":\t" + str(value) + " " + unit + line_ending)
  108. else:
  109. func(description + ":\t" + str(value) + line_ending)
  110. def print_statistics(self):
  111. """
  112. Prints the basic file statistics to the terminal.
  113. """
  114. print("\nPCAP FILE INFORMATION ------------------------------")
  115. Statistics.write_list(self.get_file_information(), print, "")
  116. print("\nGENERAL FILE STATISTICS ----------------------------")
  117. Statistics.write_list(self.get_general_file_statistics(), print, "")
  118. print("\n")
  119. def calculate_entropy(self, frequency:list, normalized:bool = False):
  120. """
  121. Calculates entropy and normalized entropy of list of elements that have specific frequency
  122. :param frequency: The frequency of the elements.
  123. :param normalized: Calculate normalized entropy
  124. :return: entropy or (entropy, normalized entropy)
  125. """
  126. entropy, normalizedEnt, n = 0, 0, 0
  127. sumFreq = sum(frequency)
  128. for i, x in enumerate(frequency):
  129. p_x = float(frequency[i] / sumFreq)
  130. if p_x > 0:
  131. n += 1
  132. entropy += - p_x * log(p_x, 2)
  133. if normalized:
  134. if log(n)>0:
  135. normalizedEnt = entropy/log(n, 2)
  136. return entropy, normalizedEnt
  137. else:
  138. return entropy
  139. def calculate_complement_packet_rates(self, pps):
  140. """
  141. Calculates the complement packet rates of the background traffic packet rates for each interval.
  142. Then normalize it to maximum boundary, which is the input parameter pps
  143. :return: normalized packet rates for each time interval.
  144. """
  145. result = self.process_db_query(
  146. "SELECT lastPktTimestamp,pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  147. # print(result)
  148. bg_interval_pps = []
  149. complement_interval_pps = []
  150. intervalsSum = 0
  151. if result:
  152. # Get the interval in seconds
  153. for i, row in enumerate(result):
  154. if i < len(result) - 1:
  155. intervalsSum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
  156. interval = intervalsSum / (len(result) - 1)
  157. # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
  158. for row in result:
  159. bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
  160. # Find max PPS
  161. maxPPS = max(bg_interval_pps, key=itemgetter(1))[1]
  162. for row in bg_interval_pps:
  163. complement_interval_pps.append((row[0], int(pps * (maxPPS - row[1]) / maxPPS)))
  164. return complement_interval_pps
  165. def get_tests_statistics(self):
  166. """
  167. Writes the calculated basic defects tests statistics into a file.
  168. """
  169. # self.stats_db._process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
  170. def count_frequncy(valuesList):
  171. values, frequency = [] , []
  172. for x in valuesList:
  173. if x in values:
  174. frequency[values.index(x)] += 1
  175. else:
  176. values.append(x)
  177. frequency.append(1)
  178. return values, frequency
  179. ####### Payload Tests #######
  180. sumPayloadCount = self.stats_db._process_user_defined_query("SELECT sum(payloadCount) FROM interval_statistics")
  181. pktCount = self.stats_db._process_user_defined_query("SELECT packetCount FROM file_statistics")
  182. if sumPayloadCount and pktCount:
  183. payloadRatio=0
  184. if(pktCount[0][0]!=0):
  185. payloadRatio = float(sumPayloadCount[0][0] / pktCount[0][0] * 100)
  186. else:
  187. payloadRatio = -1
  188. ####### TCP checksum Tests #######
  189. incorrectChecksumCount = self.stats_db._process_user_defined_query("SELECT sum(incorrectTCPChecksumCount) FROM interval_statistics")
  190. correctChecksumCount = self.stats_db._process_user_defined_query("SELECT avg(correctTCPChecksumCount) FROM interval_statistics")
  191. if incorrectChecksumCount and correctChecksumCount:
  192. incorrectChecksumRatio=0
  193. if(incorrectChecksumCount[0][0] + correctChecksumCount[0][0])!=0:
  194. incorrectChecksumRatio = float(incorrectChecksumCount[0][0] / (incorrectChecksumCount[0][0] + correctChecksumCount[0][0] ) * 100)
  195. else:
  196. incorrectChecksumRatio = -1
  197. ####### IP Src & Dst Tests #######
  198. result = self.stats_db._process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
  199. data, srcFrequency, dstFrequency = [], [], []
  200. if result:
  201. for row in result:
  202. srcFrequency.append(row[1])
  203. dstFrequency.append(row[2])
  204. ipSrcEntropy, ipSrcNormEntropy = self.calculate_entropy(srcFrequency, True)
  205. ipDstEntropy, ipDstNormEntropy = self.calculate_entropy(dstFrequency, True)
  206. newIPCount = self.stats_db._process_user_defined_query("SELECT newIPCount FROM interval_statistics")
  207. ipNovelsPerInterval, ipNovelsPerIntervalFrequency = count_frequncy(newIPCount)
  208. ipNoveltyDistEntropy = self.calculate_entropy(ipNovelsPerIntervalFrequency)
  209. ####### Ports Tests #######
  210. port0Count = self.stats_db._process_user_defined_query("SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
  211. if not port0Count[0][0]:
  212. port0Count = 0
  213. else:
  214. port0Count = port0Count[0][0]
  215. reservedPortCount = self.stats_db._process_user_defined_query(
  216. "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")# could be extended
  217. if not reservedPortCount[0][0]:
  218. reservedPortCount = 0
  219. else:
  220. reservedPortCount = reservedPortCount[0][0]
  221. ####### TTL Tests #######
  222. result = self.stats_db._process_user_defined_query("SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  223. data, frequency = [], []
  224. for row in result:
  225. frequency.append(row[1])
  226. ttlEntropy, ttlNormEntropy = self.calculate_entropy(frequency,True)
  227. newTTLCount = self.stats_db._process_user_defined_query("SELECT newTTLCount FROM interval_statistics")
  228. ttlNovelsPerInterval, ttlNovelsPerIntervalFrequency = count_frequncy(newTTLCount)
  229. ttlNoveltyDistEntropy = self.calculate_entropy(ttlNovelsPerIntervalFrequency)
  230. ####### Window Size Tests #######
  231. result = self.stats_db._process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
  232. data, frequency = [], []
  233. for row in result:
  234. frequency.append(row[1])
  235. winEntropy, winNormEntropy = self.calculate_entropy(frequency, True)
  236. newWinSizeCount = self.stats_db._process_user_defined_query("SELECT newWinSizeCount FROM interval_statistics")
  237. winNovelsPerInterval, winNovelsPerIntervalFrequency = count_frequncy(newWinSizeCount)
  238. winNoveltyDistEntropy = self.calculate_entropy(winNovelsPerIntervalFrequency)
  239. ####### ToS Tests #######
  240. result = self.stats_db._process_user_defined_query(
  241. "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
  242. data, frequency = [], []
  243. for row in result:
  244. frequency.append(row[1])
  245. tosEntropy, tosNormEntropy = self.calculate_entropy(frequency, True)
  246. newToSCount = self.stats_db._process_user_defined_query("SELECT newToSCount FROM interval_statistics")
  247. tosNovelsPerInterval, tosNovelsPerIntervalFrequency = count_frequncy(newToSCount)
  248. tosNoveltyDistEntropy = self.calculate_entropy(tosNovelsPerIntervalFrequency)
  249. ####### MSS Tests #######
  250. result = self.stats_db._process_user_defined_query(
  251. "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  252. data, frequency = [], []
  253. for row in result:
  254. frequency.append(row[1])
  255. mssEntropy, mssNormEntropy = self.calculate_entropy(frequency, True)
  256. newMSSCount = self.stats_db._process_user_defined_query("SELECT newMSSCount FROM interval_statistics")
  257. mssNovelsPerInterval, mssNovelsPerIntervalFrequency = count_frequncy(newMSSCount)
  258. mssNoveltyDistEntropy = self.calculate_entropy(mssNovelsPerIntervalFrequency)
  259. result = self.stats_db._process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
  260. # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
  261. if not result[0][0]:
  262. result = 0
  263. else:
  264. result = result[0][0]
  265. bigMSS = (result / sum(frequency)) * 100
  266. output = []
  267. if self.do_extra_tests:
  268. output = [("Payload ratio", payloadRatio, "%"),
  269. ("Incorrect TCP checksum ratio", incorrectChecksumRatio, "%")]
  270. output = output + [("# IP addresses", sum([x[0] for x in newIPCount]), ""),
  271. ("IP Src Entropy", ipSrcEntropy, ""),
  272. ("IP Src Normalized Entropy", ipSrcNormEntropy, ""),
  273. ("IP Dst Entropy", ipDstEntropy, ""),
  274. ("IP Dst Normalized Entropy", ipDstNormEntropy, ""),
  275. ("IP Novelty Distribution Entropy", ipNoveltyDistEntropy, ""),
  276. ("# TTL values", sum([x[0] for x in newTTLCount]), ""),
  277. ("TTL Entropy", ttlEntropy, ""),
  278. ("TTL Normalized Entropy", ttlNormEntropy, ""),
  279. ("TTL Novelty Distribution Entropy", ttlNoveltyDistEntropy, ""),
  280. ("# WinSize values", sum([x[0] for x in newWinSizeCount]), ""),
  281. ("WinSize Entropy", winEntropy, ""),
  282. ("WinSize Normalized Entropy", winNormEntropy, ""),
  283. ("WinSize Novelty Distribution Entropy", winNoveltyDistEntropy, ""),
  284. ("# ToS values", sum([x[0] for x in newToSCount]), ""),
  285. ("ToS Entropy", tosEntropy, ""),
  286. ("ToS Normalized Entropy", tosNormEntropy, ""),
  287. ("ToS Novelty Distribution Entropy", tosNoveltyDistEntropy, ""),
  288. ("# MSS values", sum([x[0] for x in newMSSCount]), ""),
  289. ("MSS Entropy", mssEntropy, ""),
  290. ("MSS Normalized Entropy", mssNormEntropy, ""),
  291. ("MSS Novelty Distribution Entropy", mssNoveltyDistEntropy, ""),
  292. ("======================","","")]
  293. # Reasoning the statistics values
  294. if self.do_extra_tests:
  295. if payloadRatio > 80:
  296. output.append(("WARNING: Too high payload ratio", payloadRatio, "%."))
  297. if payloadRatio < 30:
  298. output.append(("WARNING: Too low payload ratio", payloadRatio, "% (Injecting attacks that are carried out in the packet payloads is not recommmanded)."))
  299. if incorrectChecksumRatio > 5:
  300. output.append(("WARNING: High incorrect TCP checksum ratio",incorrectChecksumRatio,"%."))
  301. if ipSrcNormEntropy > 0.65:
  302. output.append(("WARNING: High IP source normalized entropy",ipSrcNormEntropy,"."))
  303. if ipSrcNormEntropy < 0.2:
  304. output.append(("WARNING: Low IP source normalized entropy", ipSrcNormEntropy, "."))
  305. if ipDstNormEntropy > 0.65:
  306. output.append(("WARNING: High IP destination normalized entropy", ipDstNormEntropy, "."))
  307. if ipDstNormEntropy < 0.2:
  308. output.append(("WARNING: Low IP destination normalized entropy", ipDstNormEntropy, "."))
  309. if ttlNormEntropy > 0.65:
  310. output.append(("WARNING: High TTL normalized entropy", ttlNormEntropy, "."))
  311. if ttlNormEntropy < 0.2:
  312. output.append(("WARNING: Low TTL normalized entropy", ttlNormEntropy, "."))
  313. if ttlNoveltyDistEntropy < 1:
  314. output.append(("WARNING: Too low TTL novelty distribution entropy", ttlNoveltyDistEntropy,
  315. "(The distribution of the novel TTL values is suspicious)."))
  316. if winNormEntropy > 0.6:
  317. output.append(("WARNING: High Window Size normalized entropy", winNormEntropy, "."))
  318. if winNormEntropy < 0.1:
  319. output.append(("WARNING: Low Window Size normalized entropy", winNormEntropy, "."))
  320. if winNoveltyDistEntropy < 4:
  321. output.append(("WARNING: Low Window Size novelty distribution entropy", winNoveltyDistEntropy,
  322. "(The distribution of the novel Window Size values is suspicious)."))
  323. if tosNormEntropy > 0.4:
  324. output.append(("WARNING: High ToS normalized entropy", tosNormEntropy, "."))
  325. if tosNormEntropy < 0.1:
  326. output.append(("WARNING: Low ToS normalized entropy", tosNormEntropy, "."))
  327. if tosNoveltyDistEntropy < 0.5:
  328. output.append(("WARNING: Low ToS novelty distribution entropy", tosNoveltyDistEntropy,
  329. "(The distribution of the novel ToS values is suspicious)."))
  330. if mssNormEntropy > 0.4:
  331. output.append(("WARNING: High MSS normalized entropy", mssNormEntropy, "."))
  332. if mssNormEntropy < 0.1:
  333. output.append(("WARNING: Low MSS normalized entropy", mssNormEntropy, "."))
  334. if mssNoveltyDistEntropy < 0.5:
  335. output.append(("WARNING: Low MSS novelty distribution entropy", mssNoveltyDistEntropy,
  336. "(The distribution of the novel MSS values is suspicious)."))
  337. if bigMSS > 50:
  338. output.append(("WARNING: High ratio of MSS > 1460", bigMSS, "% (High fragmentation rate in Ethernet)."))
  339. if port0Count > 0:
  340. output.append(("WARNING: Port number 0 is used in ",port0Count,"packets (awkward-looking port)."))
  341. if reservedPortCount > 0:
  342. output.append(("WARNING: Reserved port numbers are used in ",reservedPortCount,"packets (uncommonly-used ports)."))
  343. return output
  344. def write_statistics_to_file(self):
  345. """
  346. Writes the calculated basic statistics into a file.
  347. """
  348. def _write_header(title: str):
  349. """
  350. Writes the section header into the open file.
  351. :param title: The section title
  352. """
  353. target.write("====================== \n")
  354. target.write(title + " \n")
  355. target.write("====================== \n")
  356. target = open(self.pcap_filepath + ".stat", 'w')
  357. target.truncate()
  358. _write_header("PCAP file information")
  359. Statistics.write_list(self.get_file_information(), target.write)
  360. _write_header("General statistics")
  361. Statistics.write_list(self.get_general_file_statistics(), target.write)
  362. _write_header("Tests statistics")
  363. Statistics.write_list(self.get_tests_statistics(), target.write)
  364. target.close()
  365. def get_capture_duration(self):
  366. """
  367. :return: The duration of the capture in seconds
  368. """
  369. return self.file_info['captureDuration']
  370. def get_pcap_timestamp_start(self):
  371. """
  372. :return: The timestamp of the first packet in the PCAP file
  373. """
  374. return self.file_info['timestampFirstPacket']
  375. def get_pcap_timestamp_end(self):
  376. """
  377. :return: The timestamp of the last packet in the PCAP file
  378. """
  379. return self.file_info['timestampLastPacket']
  380. def get_pps_sent(self, ip_address: str):
  381. """
  382. Calculates the sent packets per seconds for a given IP address.
  383. :param ip_address: The IP address whose packets per second should be calculated
  384. :return: The sent packets per seconds for the given IP address
  385. """
  386. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  387. (ip_address,))
  388. capture_duration = float(self.get_capture_duration())
  389. return int(float(packets_sent) / capture_duration)
  390. def get_pps_received(self, ip_address: str):
  391. """
  392. Calculate the packets per second received for a given IP address.
  393. :param ip_address: The IP address used for the calculation
  394. :return: The number of packets per second received
  395. """
  396. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  397. False,
  398. (ip_address,))
  399. capture_duration = float(self.get_capture_duration())
  400. return int(float(packets_received) / capture_duration)
  401. def get_packet_count(self):
  402. """
  403. :return: The number of packets in the loaded PCAP file
  404. """
  405. return self.file_info['packetCount']
  406. def get_most_used_ip_address(self):
  407. """
  408. :return: The IP address/addresses with the highest sum of packets sent and received
  409. """
  410. return self.process_db_query("most_used(ipAddress)")
  411. def get_ttl_distribution(self, ipAddress: str):
  412. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ipAddress + '"')
  413. result_dict = {key: value for (key, value) in result}
  414. return result_dict
  415. def get_mss_distribution(self, ipAddress: str):
  416. result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ipAddress + '"')
  417. result_dict = {key: value for (key, value) in result}
  418. return result_dict
  419. def get_win_distribution(self, ipAddress: str):
  420. result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ipAddress + '"')
  421. result_dict = {key: value for (key, value) in result}
  422. return result_dict
  423. def get_tos_distribution(self, ipAddress: str):
  424. result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ipAddress + '"')
  425. result_dict = {key: value for (key, value) in result}
  426. return result_dict
  427. def get_random_ip_address(self, count: int = 1):
  428. """
  429. :param count: The number of IP addreses to return
  430. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of randomly
  431. chosen IP addresses
  432. """
  433. if count == 1:
  434. return self.process_db_query("random(all(ipAddress))")
  435. else:
  436. ip_address_list = []
  437. for i in range(0, count):
  438. ip_address_list.append(self.process_db_query("random(all(ipAddress))"))
  439. return ip_address_list
  440. def get_mac_address(self, ipAddress: str):
  441. """
  442. :return: The MAC address used in the dataset for the given IP address.
  443. """
  444. return self.process_db_query('macAddress(ipAddress=' + ipAddress + ")")
  445. def get_most_used_mss(self, ipAddress: str):
  446. """
  447. :param ipAddress: The IP address whose used MSS should be determined
  448. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  449. then None is returned
  450. """
  451. mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ipAddress + '" ORDER BY mssCount DESC LIMIT 1')
  452. if isinstance(mss_value, int):
  453. return mss_value
  454. else:
  455. return None
  456. def get_most_used_ttl(self, ipAddress: str):
  457. """
  458. :param ipAddress: The IP address whose used TTL should be determined
  459. :return: The TTL value used by the IP address, or if the IP addresses never specified a TTL,
  460. then None is returned
  461. """
  462. ttl_value = self.process_db_query(
  463. 'SELECT ttlValue from ip_ttl WHERE ipAddress="' + ipAddress + '" ORDER BY ttlCount DESC LIMIT 1')
  464. if isinstance(ttl_value, int):
  465. return ttl_value
  466. else:
  467. return None
  468. def get_in_degree(self):
  469. """
  470. determines the in-degree for each ipAddress, i.e. for every IP the count of ipAddresses it has received packets from
  471. :return: a list, each entry consists of one IPAddress and its associated in-degree
  472. """
  473. in_degree_raw = self.stats_db._process_user_defined_query(
  474. "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressA WHERE portDirection=\'in\' AND portNumber = portA GROUP BY ipAddress " +
  475. "UNION " +
  476. "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressB WHERE portDirection=\'in\' AND portNumber = portB GROUP BY ipAddress")
  477. #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
  478. in_degree = self.filter_multiples(in_degree_raw)
  479. return in_degree
  480. def get_out_degree(self):
  481. """
  482. determines the out-degree for each ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
  483. :return: a list, each entry consists of one IPAddress and its associated out-degree
  484. """
  485. out_degree_raw = self.stats_db._process_user_defined_query(
  486. "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA GROUP BY ipAddress " +
  487. "UNION " +
  488. "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB GROUP BY ipAddress")
  489. #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
  490. out_degree = self.filter_multiples(out_degree_raw)
  491. return out_degree
  492. def get_overall_degree(self):
  493. """
  494. determines the overall-degree for each ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
  495. :return: a list, each entry consists of one IPAddress and its associated overall-degree
  496. """
  497. out_degrees = self.get_out_degree()
  498. in_degrees = self.get_in_degree()
  499. overall_degrees = []
  500. processed = {} # Dict, taking an IP Address and returning True, if the IP has already been processed and added to overall_degree
  501. # initialize values of the dict for in_degrees, this is important for error-free checking whether there are not processed IPs
  502. # for out_degrees this can be done without an additional loop
  503. for inD in in_degrees:
  504. processed[inD[0]] = False
  505. for outD in out_degrees:
  506. ip_out = outD[0]
  507. processed[ip_out] = False
  508. # add the sum of degrees for all IPs that appear in both lists
  509. for inD in in_degrees:
  510. ip_in = inD[0]
  511. if ip_out == ip_in:
  512. # same IPAddress -> append sum of degrees
  513. overall_degrees.append((ip_out, outD[1] + inD[1]))
  514. processed[ip_out] = True
  515. if not processed[ip_out]:
  516. # if IP only appears in out_degree list -> just append the value
  517. overall_degrees.append(outD)
  518. processed[outD[0]] = True
  519. # add remaining IPs, which did not appear in out_degree
  520. for inD in in_degrees:
  521. if not processed[inD[0]]:
  522. overall_degrees.append(inD)
  523. return overall_degrees
  524. def filter_multiples(self, entries):
  525. """
  526. helper function, for get_out_degree and get_in_degree
  527. filters the given list for duplicate IpAddresses and, if duplciates are present, accumulates their values
  528. :param entries: list, each entry consists of an ipAddress and a numeric value
  529. :return: a filtered list, without duplicate ipAddresses
  530. """
  531. filtered_entries = []
  532. done = []
  533. for p1 in entries:
  534. added = False
  535. if p1 in done:
  536. continue
  537. for p2 in entries:
  538. if p1[0] == p2[0] and p1 != p2:
  539. filtered_entries.append((p1[0], p1[1] + p2[1]))
  540. done.append(p1)
  541. done.append(p2)
  542. added = True
  543. break
  544. if not added:
  545. filtered_entries.append(p1)
  546. return filtered_entries
  547. def get_avg_delay_local_ext(self):
  548. """
  549. Calculates the average delay of a packet for external and local communication, based on the tcp handshakes
  550. :return: tuple consisting of avg delay for local and external communication, (local, external)
  551. """
  552. conv_delays = self.stats_db._process_user_defined_query("SELECT ipAddressA, ipAddressB, avgDelay FROM conv_statistics")
  553. if(conv_delays):
  554. external_conv = []
  555. local_conv = []
  556. for conv in conv_delays:
  557. IPA = IPAddress.parse(conv[0])
  558. IPB = IPAddress.parse(conv[1])
  559. #split into local and external conversations
  560. if(not IPA.is_private() or not IPB.is_private()):
  561. external_conv.append(conv)
  562. else:
  563. local_conv.append(conv)
  564. # calculate avg local and external delay by summing up the respective delays and dividing them by the number of conversations
  565. avg_delay_external = 0.0
  566. avg_delay_local = 0.0
  567. if(local_conv):
  568. for conv in local_conv:
  569. avg_delay_local += conv[2]
  570. avg_delay_local = (avg_delay_local/len(local_conv)) * 0.001 #ms
  571. else:
  572. # no local conversations in statistics found
  573. avg_delay_local = 0.06
  574. if(external_conv):
  575. for conv in external_conv:
  576. avg_delay_external += conv[2]
  577. avg_delay_external = (avg_delay_external/len(external_conv)) * 0.001 #ms
  578. else:
  579. # no external conversations in statistics found
  580. avg_delay_external = 0.15
  581. else:
  582. #if no statistics were found, use these numbers
  583. avg_delay_external = 0.15
  584. avg_delay_local = 0.06
  585. return avg_delay_local, avg_delay_external
  586. def get_statistics_database(self):
  587. """
  588. :return: A reference to the statistics database object
  589. """
  590. return self.stats_db
  591. def process_db_query(self, query_string_in: str, print_results: bool = False):
  592. """
  593. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  594. query.
  595. :param query_string_in: The query to be processed
  596. :param print_results: Indicates whether the results should be printed to terminal
  597. :return: The result of the query
  598. """
  599. return self.stats_db.process_db_query(query_string_in, print_results)
  600. def is_query(self, value: str):
  601. """
  602. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  603. :param value: The string to be checked
  604. :return: True if the string is recognized as a query, otherwise False.
  605. """
  606. if not isinstance(value, str):
  607. return False
  608. else:
  609. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  610. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  611. def calculate_standard_deviation(self, lst):
  612. """
  613. Calculates the standard deviation of a list of numbers.
  614. :param lst: The list of numbers to calculate its SD.
  615. """
  616. num_items = len(lst)
  617. mean = sum(lst) / num_items
  618. differences = [x - mean for x in lst]
  619. sq_differences = [d ** 2 for d in differences]
  620. ssd = sum(sq_differences)
  621. variance = ssd / num_items
  622. sd = sqrt(variance)
  623. return sd
  624. def plot_statistics(self, format: str = 'pdf'): #'png'
  625. """
  626. Plots the statistics associated with the dataset.
  627. :param format: The format to be used to save the statistics diagrams.
  628. """
  629. def plot_distribution(queryOutput, title, xLabel, yLabel, file_ending: str):
  630. plt.gcf().clear()
  631. graphx, graphy = [], []
  632. for row in queryOutput:
  633. graphx.append(row[0])
  634. graphy.append(row[1])
  635. plt.autoscale(enable=True, axis='both')
  636. plt.title(title)
  637. plt.xlabel(xLabel)
  638. plt.ylabel(yLabel)
  639. width = 0.1
  640. plt.xlim([0, max(graphx)])
  641. plt.grid(True)
  642. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  643. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  644. plt.savefig(out,dpi=500)
  645. return out
  646. def plot_ttl(file_ending: str):
  647. queryOutput = self.stats_db._process_user_defined_query(
  648. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  649. title = "TTL Distribution"
  650. xLabel = "TTL Value"
  651. yLabel = "Number of Packets"
  652. if queryOutput:
  653. return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
  654. def plot_mss(file_ending: str):
  655. queryOutput = self.stats_db._process_user_defined_query(
  656. "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
  657. title = "MSS Distribution"
  658. xLabel = "MSS Value"
  659. yLabel = "Number of Packets"
  660. if queryOutput:
  661. return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
  662. def plot_win(file_ending: str):
  663. queryOutput = self.stats_db._process_user_defined_query(
  664. "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
  665. title = "Window Size Distribution"
  666. xLabel = "Window Size"
  667. yLabel = "Number of Packets"
  668. if queryOutput:
  669. return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
  670. def plot_protocol(file_ending: str):
  671. plt.gcf().clear()
  672. result = self.stats_db._process_user_defined_query(
  673. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  674. if (result):
  675. graphx, graphy = [], []
  676. for row in result:
  677. graphx.append(row[0])
  678. graphy.append(row[1])
  679. plt.autoscale(enable=True, axis='both')
  680. plt.title("Protocols Distribution")
  681. plt.xlabel('Protocols')
  682. plt.ylabel('Number of Packets')
  683. width = 0.1
  684. plt.xlim([0, len(graphx)])
  685. plt.grid(True)
  686. # Protocols' names on x-axis
  687. x = range(0,len(graphx))
  688. my_xticks = graphx
  689. plt.xticks(x, my_xticks)
  690. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  691. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  692. plt.savefig(out,dpi=500)
  693. return out
  694. else:
  695. print("Error plot protocol: No protocol values found!")
  696. def plot_port(file_ending: str):
  697. plt.gcf().clear()
  698. result = self.stats_db._process_user_defined_query(
  699. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  700. graphx, graphy = [], []
  701. for row in result:
  702. graphx.append(row[0])
  703. graphy.append(row[1])
  704. plt.autoscale(enable=True, axis='both')
  705. plt.title("Ports Distribution")
  706. plt.xlabel('Ports Numbers')
  707. plt.ylabel('Number of Packets')
  708. width = 0.1
  709. plt.xlim([0, max(graphx)])
  710. plt.grid(True)
  711. plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  712. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  713. plt.savefig(out,dpi=500)
  714. return out
  715. # This distribution is not drawable for big datasets
  716. def plot_ip_src(file_ending: str):
  717. plt.gcf().clear()
  718. result = self.stats_db._process_user_defined_query(
  719. "SELECT ipAddress, pktsSent FROM ip_statistics")
  720. graphx, graphy = [], []
  721. for row in result:
  722. graphx.append(row[0])
  723. graphy.append(row[1])
  724. plt.autoscale(enable=True, axis='both')
  725. plt.title("Source IP Distribution")
  726. plt.xlabel('Source IP')
  727. plt.ylabel('Number of Packets')
  728. width = 0.1
  729. plt.xlim([0, len(graphx)])
  730. plt.grid(True)
  731. # IPs on x-axis
  732. x = range(0, len(graphx))
  733. my_xticks = graphx
  734. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  735. plt.tight_layout()
  736. # limit the number of xticks
  737. plt.locator_params(axis='x', nbins=20)
  738. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  739. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  740. plt.savefig(out, dpi=500)
  741. return out
  742. # This distribution is not drawable for big datasets
  743. def plot_ip_dst(file_ending: str):
  744. plt.gcf().clear()
  745. result = self.stats_db._process_user_defined_query(
  746. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  747. graphx, graphy = [], []
  748. for row in result:
  749. graphx.append(row[0])
  750. graphy.append(row[1])
  751. plt.autoscale(enable=True, axis='both')
  752. plt.title("Destination IP Distribution")
  753. plt.xlabel('Destination IP')
  754. plt.ylabel('Number of Packets')
  755. width = 0.1
  756. plt.xlim([0, len(graphx)])
  757. plt.grid(True)
  758. # IPs on x-axis
  759. x = range(0, len(graphx))
  760. my_xticks = graphx
  761. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  762. plt.tight_layout()
  763. # limit the number of xticks
  764. plt.locator_params(axis='x', nbins=20)
  765. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  766. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  767. plt.savefig(out, dpi=500)
  768. return out
  769. def plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending: str):
  770. plt.gcf().clear()
  771. graphx, graphy = [], []
  772. for row in queryOutput:
  773. graphx.append(row[0])
  774. graphy.append(row[1])
  775. plt.autoscale(enable=True, axis='both')
  776. plt.title(title)
  777. plt.xlabel(xLabel)
  778. plt.ylabel(yLabel)
  779. width = 0.5
  780. plt.xlim([0, len(graphx)])
  781. plt.grid(True)
  782. # timestamp on x-axis
  783. x = range(0, len(graphx))
  784. # limit the number of xticks
  785. plt.locator_params(axis='x', nbins=20)
  786. plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
  787. out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
  788. plt.savefig(out, dpi=500)
  789. return out
  790. def plot_interval_pktCount(file_ending: str):
  791. queryOutput = self.stats_db._process_user_defined_query(
  792. "SELECT lastPktTimestamp, pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
  793. title = "Packet Rate"
  794. xLabel = "Time Interval"
  795. yLabel = "Number of Packets"
  796. if queryOutput:
  797. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  798. def plot_interval_ip_src_ent(file_ending: str):
  799. queryOutput = self.stats_db._process_user_defined_query(
  800. "SELECT lastPktTimestamp, ipSrcEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  801. title = "Source IP Entropy"
  802. xLabel = "Time Interval"
  803. yLabel = "Entropy"
  804. if queryOutput:
  805. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  806. def plot_interval_ip_dst_ent(file_ending: str):
  807. queryOutput = self.stats_db._process_user_defined_query(
  808. "SELECT lastPktTimestamp, ipDstEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  809. title = "Destination IP Entropy"
  810. xLabel = "Time Interval"
  811. yLabel = "Entropy"
  812. if queryOutput:
  813. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  814. def plot_interval_new_ip(file_ending: str):
  815. queryOutput = self.stats_db._process_user_defined_query(
  816. "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
  817. title = "IP Novelty Distribution"
  818. xLabel = "Time Interval"
  819. yLabel = "Novel values count"
  820. if queryOutput:
  821. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  822. def plot_interval_new_port(file_ending: str):
  823. queryOutput = self.stats_db._process_user_defined_query(
  824. "SELECT lastPktTimestamp, newPortCount FROM interval_statistics ORDER BY lastPktTimestamp")
  825. title = "Port Novelty Distribution"
  826. xLabel = "Time Interval"
  827. yLabel = "Novel values count"
  828. if queryOutput:
  829. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  830. def plot_interval_new_ttl(file_ending: str):
  831. queryOutput = self.stats_db._process_user_defined_query(
  832. "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
  833. title = "TTL Novelty Distribution"
  834. xLabel = "Time Interval"
  835. yLabel = "Novel values count"
  836. if queryOutput:
  837. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  838. def plot_interval_new_tos(file_ending: str):
  839. queryOutput = self.stats_db._process_user_defined_query(
  840. "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  841. title = "ToS Novelty Distribution"
  842. xLabel = "Time Interval"
  843. yLabel = "Novel values count"
  844. if queryOutput:
  845. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  846. def plot_interval_new_win_size(file_ending: str):
  847. queryOutput = self.stats_db._process_user_defined_query(
  848. "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
  849. title = "Window Size Novelty Distribution"
  850. xLabel = "Time Interval"
  851. yLabel = "Novel values count"
  852. if queryOutput:
  853. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  854. def plot_interval_new_mss(file_ending: str):
  855. queryOutput = self.stats_db._process_user_defined_query(
  856. "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
  857. title = "MSS Novelty Distribution"
  858. xLabel = "Time Interval"
  859. yLabel = "Novel values count"
  860. if queryOutput:
  861. return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
  862. def plot_interval_ip_dst_cum_ent(file_ending: str):
  863. plt.gcf().clear()
  864. result = self.stats_db._process_user_defined_query(
  865. "SELECT lastPktTimestamp, ipDstCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  866. graphx, graphy = [], []
  867. for row in result:
  868. graphx.append(row[0])
  869. graphy.append(row[1])
  870. # If entropy was not calculated do not plot the graph
  871. if graphy[0] != -1:
  872. plt.autoscale(enable=True, axis='both')
  873. plt.title("Destination IP Cumulative Entropy")
  874. # plt.xlabel('Timestamp')
  875. plt.xlabel('Time Interval')
  876. plt.ylabel('Entropy')
  877. plt.xlim([0, len(graphx)])
  878. plt.grid(True)
  879. # timestamp on x-axis
  880. x = range(0, len(graphx))
  881. # my_xticks = graphx
  882. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  883. # plt.tight_layout()
  884. # limit the number of xticks
  885. plt.locator_params(axis='x', nbins=20)
  886. plt.plot(x, graphy, 'r')
  887. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-cum-ent' + file_ending)
  888. plt.savefig(out, dpi=500)
  889. return out
  890. def plot_interval_ip_src_cum_ent(file_ending: str):
  891. plt.gcf().clear()
  892. result = self.stats_db._process_user_defined_query(
  893. "SELECT lastPktTimestamp, ipSrcCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
  894. graphx, graphy = [], []
  895. for row in result:
  896. graphx.append(row[0])
  897. graphy.append(row[1])
  898. # If entropy was not calculated do not plot the graph
  899. if graphy[0] != -1:
  900. plt.autoscale(enable=True, axis='both')
  901. plt.title("Source IP Cumulative Entropy")
  902. # plt.xlabel('Timestamp')
  903. plt.xlabel('Time Interval')
  904. plt.ylabel('Entropy')
  905. plt.xlim([0, len(graphx)])
  906. plt.grid(True)
  907. # timestamp on x-axis
  908. x = range(0, len(graphx))
  909. # my_xticks = graphx
  910. # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  911. # plt.tight_layout()
  912. # limit the number of xticks
  913. plt.locator_params(axis='x', nbins=20)
  914. plt.plot(x, graphy, 'r')
  915. out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-cum-ent' + file_ending)
  916. plt.savefig(out, dpi=500)
  917. return out
  918. def plot_in_degree(file_ending: str):
  919. """
  920. Creates a Plot, visualizing the in-degree for every IP Address
  921. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  922. :return: A filepath to the file containing the created plot
  923. """
  924. plt.gcf().clear()
  925. # retrieve data
  926. in_degree = self.get_in_degree()
  927. if(in_degree):
  928. graphx, graphy = [], []
  929. for entry in in_degree:
  930. # degree values
  931. graphx.append(entry[1])
  932. # IP labels
  933. graphy.append(entry[0])
  934. # set labels
  935. plt.title("Indegree per IP Address")
  936. plt.ylabel('IpAddress')
  937. plt.xlabel('Indegree')
  938. #set width of the bars
  939. width = 0.3
  940. # set scalings
  941. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  942. #set limits of the axis
  943. plt.ylim([0, len(graphy)])
  944. plt.xlim([0, max(graphx) + 10])
  945. # display numbers at each bar
  946. for i, v in enumerate(graphx):
  947. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  948. # display grid for better visuals
  949. plt.grid(True)
  950. # plot the bar
  951. labels = graphy
  952. graphy = list(range(len(graphx)))
  953. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  954. plt.yticks(graphy, labels)
  955. out = self.pcap_filepath.replace('.pcap', '_in_degree' + file_ending)
  956. plt.tight_layout()
  957. plt.savefig(out,dpi=500)
  958. return out
  959. else:
  960. print("Error: No statistics Information for plotting out-degrees found")
  961. def plot_out_degree(file_ending: str):
  962. """
  963. Creates a Plot, visualizing the out-degree for every IP Address
  964. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  965. :return: A filepath to the file containing the created plot
  966. """
  967. plt.gcf().clear()
  968. # retrieve data
  969. out_degree = self.get_out_degree()
  970. if(out_degree):
  971. graphx, graphy = [], []
  972. for entry in out_degree:
  973. # degree values
  974. graphx.append(entry[1])
  975. # IP labels
  976. graphy.append(entry[0])
  977. # set labels
  978. plt.title("Outdegree per IP Address")
  979. plt.ylabel('IpAddress')
  980. plt.xlabel('Outdegree')
  981. #set width of the bars
  982. width = 0.3
  983. # set scalings
  984. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  985. #set limits of the axis
  986. plt.ylim([0, len(graphy)])
  987. plt.xlim([0, max(graphx) + 10])
  988. # display numbers at each bar
  989. for i, v in enumerate(graphx):
  990. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  991. # display grid for better visuals
  992. plt.grid(True)
  993. # plot the bar
  994. labels = graphy
  995. graphy = list(range(len(graphx)))
  996. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  997. plt.yticks(graphy, labels)
  998. out = self.pcap_filepath.replace('.pcap', '_out_degree' + file_ending)
  999. plt.tight_layout()
  1000. plt.savefig(out,dpi=500)
  1001. return out
  1002. else:
  1003. print("Error: No statistics Information for plotting out-degrees found")
  1004. def plot_overall_degree(file_ending: str):
  1005. """
  1006. Creates a Plot, visualizing the overall-degree for every IP Address
  1007. :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
  1008. :return: A filepath to the file containing the created plot
  1009. """
  1010. plt.gcf().clear()
  1011. # retrieve data
  1012. overall_degree = self.get_overall_degree()
  1013. if(overall_degree):
  1014. graphx, graphy = [], []
  1015. for entry in overall_degree:
  1016. # degree values
  1017. graphx.append(entry[1])
  1018. # IP labels
  1019. graphy.append(entry[0])
  1020. # set labels
  1021. plt.title("Overalldegree per IP Address")
  1022. plt.ylabel('IpAddress')
  1023. plt.xlabel('Overalldegree')
  1024. #set width of the bars
  1025. width = 0.3
  1026. # set scalings
  1027. plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5)) # these proportions just worked well
  1028. #set limits of the axis
  1029. plt.ylim([0, len(graphy)])
  1030. plt.xlim([0, max(graphx) + 10])
  1031. # display numbers at each bar
  1032. for i, v in enumerate(graphx):
  1033. plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
  1034. # display grid for better visuals
  1035. plt.grid(True)
  1036. # plot the bar
  1037. labels = graphy
  1038. graphy = list(range(len(graphx)))
  1039. plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  1040. plt.yticks(graphy, labels)
  1041. out = self.pcap_filepath.replace('.pcap', '_overall_degree' + file_ending)
  1042. plt.tight_layout()
  1043. plt.savefig(out,dpi=500)
  1044. return out
  1045. else:
  1046. print("Error: No statistics Information for plotting overall-degrees found")
  1047. def plot_big_comm_interval_stat(attr:str, table:str, title:str, xlabel:str, suffix:str):
  1048. """
  1049. Plots the desired statistc per connection as horizontal bar plot.
  1050. Included are 'half-open' connections, where only one packet is exchanged.
  1051. The given statistics table has to have at least the attributes 'ipAddressA', 'portA', 'ipAddressB',
  1052. 'portB' and the specified additional attribute.
  1053. Note: there may be cutoff/scaling problems within the plot if there is too little data.
  1054. :param attr: The desired statistic, named with respect to its attribute in the given statistics table
  1055. :param table: The statistics table
  1056. :param title: The title of the created plot
  1057. :param xlabel: The name of the x-axis of the created plot
  1058. :param suffix: The suffix of the created file, including file extension
  1059. :return: A filepath to the file containing the created plot
  1060. """
  1061. plt.gcf().clear()
  1062. result = self.stats_db._process_user_defined_query(
  1063. "SELECT ipAddressA, portA, ipAddressB, portB, %s FROM %s" % (attr, table))
  1064. if (result):
  1065. graphy, graphx = [], []
  1066. # plot data in descending order
  1067. result = sorted(result, key=lambda row: row[4])
  1068. # compute plot data
  1069. for i, row in enumerate(result):
  1070. addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
  1071. # adjust the justification of strings to improve appearance
  1072. len_max = max(len(addr1), len(addr2))
  1073. addr1 = addr1.ljust(len_max)
  1074. addr2 = addr2.ljust(len_max)
  1075. # add plot data
  1076. graphy.append("%s\n%s" % (addr1, addr2))
  1077. graphx.append(row[4])
  1078. # compute plot height in inches
  1079. dist_mult_height, dist_mult_width = 0.55, 0.07 # these values turned out to work well
  1080. plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
  1081. title_distance = 1 + 0.012*52.8/plt_height # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
  1082. # have x axis and its label appear at the top (instead of bottom)
  1083. fig, ax = plt.subplots()
  1084. ax.xaxis.tick_top()
  1085. ax.xaxis.set_label_position("top")
  1086. # set additional plot parameters
  1087. plt.title(title, y=title_distance)
  1088. plt.xlabel(xlabel)
  1089. plt.ylabel('Connection')
  1090. width = 0.5
  1091. plt.grid(True)
  1092. plt.gca().margins(y=0) # removes the space between data and x-axis within the plot
  1093. plt.gcf().set_size_inches(plt_width, plt_height) # set plot size
  1094. # plot the above data, first use plain numbers as graphy to maintain sorting
  1095. plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
  1096. # now change the y numbers to the respective address labels
  1097. plt.yticks(range(len(graphy)), graphy)
  1098. # try to use tight layout to cut off unnecessary space
  1099. try:
  1100. plt.tight_layout(pad=4)
  1101. except ValueError:
  1102. pass
  1103. # save created figure
  1104. out = self.pcap_filepath.replace('.pcap', suffix)
  1105. plt.savefig(out, dpi=500)
  1106. return out
  1107. def plot_packets_per_connection(file_ending: str):
  1108. """
  1109. Plots the total number of exchanged packets per connection.
  1110. :param file_ending: The file extension for the output file containing the plot
  1111. :return: A filepath to the file containing the created plot
  1112. """
  1113. title = 'Number of exchanged packets per connection'
  1114. suffix = '_plot-PktCount per Connection Distribution' + file_ending
  1115. # plot data and return outpath
  1116. return plot_big_comm_interval_stat("pktsCount", "conv_statistics_stateless", title, "Number of packets", suffix)
  1117. def plot_avg_pkts_per_comm_interval(file_ending: str):
  1118. """
  1119. Plots the average number of exchanged packets per communication interval for every connection.
  1120. :param file_ending: The file extension for the output file containing the plot
  1121. :return: A filepath to the file containing the created plot
  1122. """
  1123. title = 'Average number of exchanged packets per communication interval'
  1124. suffix = '_plot-Avg PktCount Communication Interval Distribution' + file_ending
  1125. # plot data and return outpath
  1126. return plot_big_comm_interval_stat("avgPktCount", "comm_interval_statistics" ,title, "Number of packets", suffix)
  1127. def plot_avg_time_between_comm_interval(file_ending: str):
  1128. """
  1129. Plots the average time between the communication intervals of every connection.
  1130. :param file_ending: The file extension for the output file containing the plot
  1131. :return: A filepath to the file containing the created plot
  1132. """
  1133. title = 'Average time between communication intervals in seconds'
  1134. suffix = '_plot-Avg Time Between Communication Intervals Distribution' + file_ending
  1135. # plot data and return outpath
  1136. return plot_big_comm_interval_stat("avgTimeBetweenIntervals", "comm_interval_statistics", title, 'Average time between intervals', suffix)
  1137. ttl_out_path = plot_ttl('.' + format)
  1138. mss_out_path = plot_mss('.' + format)
  1139. win_out_path = plot_win('.' + format)
  1140. protocol_out_path = plot_protocol('.' + format)
  1141. plot_interval_pktCount = plot_interval_pktCount('.' + format)
  1142. plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + format)
  1143. plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + format)
  1144. plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + format)
  1145. plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + format)
  1146. plot_interval_new_ip = plot_interval_new_ip('.' + format)
  1147. plot_interval_new_port = plot_interval_new_port('.' + format)
  1148. plot_interval_new_ttl = plot_interval_new_ttl('.' + format)
  1149. plot_interval_new_tos = plot_interval_new_tos('.' + format)
  1150. plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
  1151. plot_interval_new_mss = plot_interval_new_mss('.' + format)
  1152. plot_packets_per_connection_out = plot_packets_per_connection('.' + format)
  1153. plot_out_degree = plot_out_degree('.' + format)
  1154. plot_in_degree = plot_in_degree('.' + format)
  1155. plot_overall_degree = plot_overall_degree('.' + format)
  1156. plot_avg_pkts_per_comm_interval_out = plot_avg_pkts_per_comm_interval('.' + format)
  1157. plot_avg_time_between_comm_interval_out = plot_avg_time_between_comm_interval('.' + format)
  1158. ## Time consuming plot
  1159. # port_out_path = plot_port('.' + format)
  1160. ## Not drawable for too many IPs
  1161. # ip_src_out_path = plot_ip_src('.' + format)
  1162. # ip_dst_out_path = plot_ip_dst('.' + format)
  1163. print("Saved plots in the input PCAP directory.")