Statistics.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579
  1. # Aidmar
  2. from scipy.spatial import distance as dist
  3. import numpy as np
  4. import os
  5. import time
  6. import ID2TLib.libpcapreader as pr
  7. import matplotlib
  8. matplotlib.use('Agg')
  9. import matplotlib.pyplot as plt
  10. from ID2TLib.PcapFile import PcapFile
  11. from ID2TLib.StatsDatabase import StatsDatabase
  12. class Statistics:
  13. def __init__(self, pcap_file: PcapFile):
  14. """
  15. Creates a new Statistics object.
  16. :param pcap_file: A reference to the PcapFile object
  17. """
  18. # Fields
  19. self.pcap_filepath = pcap_file.pcap_file_path
  20. self.pcap_proc = None
  21. # Create folder for statistics database if required
  22. self.path_db = pcap_file.get_db_path()
  23. path_dir = os.path.dirname(self.path_db)
  24. if not os.path.isdir(path_dir):
  25. os.makedirs(path_dir)
  26. # Class instances
  27. self.stats_db = StatsDatabase(self.path_db)
  28. def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool):
  29. """
  30. Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
  31. statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
  32. statistics are gathered directly from the existing database.
  33. :param flag_write_file: Indicates whether the statistics should be written addiotionally into a text file (True)
  34. or not (False)
  35. :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
  36. :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
  37. """
  38. # Load pcap and get loading time
  39. time_start = time.clock()
  40. # Inform user about recalculation of statistics and its reason
  41. if flag_recalculate_stats:
  42. print("Flag -r/--recalculate found. Recalculating statistics.")
  43. # Recalculate statistics if database not exists OR param -r/--recalculate was provided
  44. if (not self.stats_db.get_db_exists()) or flag_recalculate_stats:
  45. self.pcap_proc = pr.pcap_processor(self.pcap_filepath)
  46. self.pcap_proc.collect_statistics()
  47. self.pcap_proc.write_to_database(self.path_db)
  48. outstring_datasource = "by PCAP file processor."
  49. else:
  50. outstring_datasource = "from statistics database."
  51. # Load statistics from database
  52. self.file_info = self.stats_db.get_file_info()
  53. time_end = time.clock()
  54. print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
  55. # Write statistics if param -e/--export provided
  56. if flag_write_file:
  57. self.write_statistics_to_file()
  58. # Print statistics if param -s/--statistics provided
  59. if flag_print_statistics:
  60. self.print_statistics()
  61. def get_file_information(self):
  62. """
  63. Returns a list of tuples, each containing a information of the file.
  64. :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
  65. """
  66. return [("Pcap file", self.pcap_filepath),
  67. ("#Packets", self.get_packet_count(), "packets"),
  68. ("Capture length", self.get_capture_duration(), "seconds"),
  69. ("Capture start", self.get_pcap_timestamp_start()),
  70. ("Capture end", self.get_pcap_timestamp_end())]
  71. def get_general_file_statistics(self):
  72. """
  73. Returns a list of tuples, each containing a file statistic.
  74. :return: a list of tuples, each consisting of (description, value, unit).
  75. """
  76. return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
  77. ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
  78. ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
  79. ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
  80. ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
  81. @staticmethod
  82. def write_list(desc_val_unit_list, func, line_ending="\n"):
  83. """
  84. Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three values
  85. and applies the function func on this string.
  86. Before generating the string, it identifies text containing a float number, casts the string to a
  87. float and rounds the value to two decimal digits.
  88. :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
  89. :param func: The function to be applied to each generated string
  90. :param line_ending: The formatting string to be applied at the end of each string
  91. """
  92. for entry in desc_val_unit_list:
  93. # Convert text containing float into float
  94. (description, value) = entry[0:2]
  95. if isinstance(value, str) and "." in value:
  96. try:
  97. value = float(value)
  98. except ValueError:
  99. pass # do nothing -> value was not a float
  100. # round float
  101. if isinstance(value, float):
  102. value = round(value, 2)
  103. # write into file
  104. if len(entry) == 3:
  105. unit = entry[2]
  106. func(description + ":\t" + str(value) + " " + unit + line_ending)
  107. else:
  108. func(description + ":\t" + str(value) + line_ending)
  109. def print_statistics(self):
  110. """
  111. Prints the basic file statistics to the terminal.
  112. """
  113. print("\nPCAP FILE INFORMATION ------------------------------")
  114. Statistics.write_list(self.get_file_information(), print, "")
  115. print("\nGENERAL FILE STATISTICS ----------------------------")
  116. Statistics.write_list(self.get_general_file_statistics(), print, "")
  117. print("\n")
  118. def write_statistics_to_file(self):
  119. """
  120. Writes the calculated basic statistics into a file.
  121. """
  122. def _write_header(title: str):
  123. """
  124. Writes the section header into the open file.
  125. :param title: The section title
  126. """
  127. target.write("====================== \n")
  128. target.write(title + " \n")
  129. target.write("====================== \n")
  130. target = open(self.pcap_filepath + ".stat", 'w')
  131. target.truncate()
  132. _write_header("PCAP file information")
  133. Statistics.write_list(self.get_file_information(), target.write)
  134. _write_header("General statistics")
  135. Statistics.write_list(self.get_general_file_statistics(), target.write)
  136. target.close()
  137. def get_capture_duration(self):
  138. """
  139. :return: The duration of the capture in seconds
  140. """
  141. return self.file_info['captureDuration']
  142. def get_pcap_timestamp_start(self):
  143. """
  144. :return: The timestamp of the first packet in the PCAP file
  145. """
  146. return self.file_info['timestampFirstPacket']
  147. def get_pcap_timestamp_end(self):
  148. """
  149. :return: The timestamp of the last packet in the PCAP file
  150. """
  151. return self.file_info['timestampLastPacket']
  152. def get_pps_sent(self, ip_address: str):
  153. """
  154. Calculates the sent packets per seconds for a given IP address.
  155. :param ip_address: The IP address whose packets per second should be calculated
  156. :return: The sent packets per seconds for the given IP address
  157. """
  158. packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
  159. (ip_address,))
  160. capture_duration = float(self.get_capture_duration())
  161. return int(float(packets_sent) / capture_duration)
  162. def get_pps_received(self, ip_address: str):
  163. """
  164. Calculate the packets per second received for a given IP address.
  165. :param ip_address: The IP address used for the calculation
  166. :return: The number of packets per second received
  167. """
  168. packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
  169. False,
  170. (ip_address,))
  171. capture_duration = float(self.get_capture_duration())
  172. return int(float(packets_received) / capture_duration)
  173. def get_packet_count(self):
  174. """
  175. :return: The number of packets in the loaded PCAP file
  176. """
  177. return self.file_info['packetCount']
  178. def get_most_used_ip_address(self):
  179. """
  180. :return: The IP address/addresses with the highest sum of packets sent and received
  181. """
  182. return self.process_db_query("most_used(ipAddress)")
  183. def get_ttl_distribution(self, ipAddress: str):
  184. result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ipAddress + '"')
  185. result_dict = {key: value for (key, value) in result}
  186. return result_dict
  187. def get_random_ip_address(self, count: int = 1):
  188. """
  189. :param count: The number of IP addreses to return
  190. :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of randomly
  191. chosen IP addresses
  192. """
  193. if count == 1:
  194. return self.process_db_query("random(all(ipAddress))")
  195. else:
  196. ip_address_list = []
  197. for i in range(0, count):
  198. ip_address_list.append(self.process_db_query("random(all(ipAddress))"))
  199. return ip_address_list
  200. def get_mac_address(self, ipAddress: str):
  201. """
  202. :return: The MAC address used in the dataset for the given IP address.
  203. """
  204. return self.process_db_query('macAddress(ipAddress=' + ipAddress + ")")
  205. def get_mss(self, ipAddress: str):
  206. """
  207. :param ipAddress: The IP address whose used MSS should be determined
  208. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  209. then None is returned
  210. """
  211. mss_value = self.process_db_query('SELECT mss from tcp_mss WHERE ipAddress="' + ipAddress + '"')
  212. if isinstance(mss_value, int):
  213. return mss_value
  214. else:
  215. return None
  216. # Aidmar
  217. def get_most_used_mss(self, ipAddress: str):
  218. """
  219. :param ipAddress: The IP address whose used MSS should be determined
  220. :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
  221. then None is returned
  222. """
  223. mss_value = self.process_db_query('SELECT mssValue from tcp_mss_dist WHERE ipAddress="' + ipAddress + '" ORDER BY mssCount DESC LIMIT 1')
  224. if isinstance(mss_value, int):
  225. return mss_value
  226. else:
  227. return None
  228. def get_statistics_database(self):
  229. """
  230. :return: A reference to the statistics database object
  231. """
  232. return self.stats_db
  233. def process_db_query(self, query_string_in: str, print_results: bool = False):
  234. """
  235. Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
  236. query.
  237. :param query_string_in: The query to be processed
  238. :param print_results: Indicates whether the results should be printed to terminal
  239. :return: The result of the query
  240. """
  241. return self.stats_db.process_db_query(query_string_in, print_results)
  242. def is_query(self, value: str):
  243. """
  244. Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
  245. :param value: The string to be checked
  246. :return: True if the string is recognized as a query, otherwise False.
  247. """
  248. if not isinstance(value, str):
  249. return False
  250. else:
  251. return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
  252. any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
  253. def plot_statistics(self, format: str = 'pdf'): #'png'):
  254. """
  255. Plots the statistics associated with the dataset prior attack injection.
  256. :param format: The format to be used to save the statistics diagrams.
  257. """
  258. def plot_ttl(file_ending: str):
  259. plt.gcf().clear()
  260. result = self.stats_db._process_user_defined_query(
  261. "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
  262. graphx, graphy = [], []
  263. for row in result:
  264. graphx.append(row[0])
  265. graphy.append(row[1])
  266. plt.autoscale(enable=True, axis='both')
  267. plt.title("TTL Distribution")
  268. plt.xlabel('TTL Value')
  269. plt.ylabel('Number of Packets')
  270. width = 0.5
  271. plt.xlim([0, max(graphx)])
  272. plt.grid(True)
  273. plt.bar(graphx, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  274. out = self.pcap_filepath.replace('.pcap', '_plot-ttl' + file_ending)
  275. plt.savefig(out,dpi=500)
  276. return out
  277. # Aidmar
  278. def plot_mss(file_ending: str):
  279. plt.gcf().clear()
  280. result = self.stats_db._process_user_defined_query(
  281. "SELECT mssValue, SUM(mssCount) FROM tcp_mss_dist GROUP BY mssValue")
  282. graphx, graphy = [], []
  283. for row in result:
  284. graphx.append(row[0])
  285. graphy.append(row[1])
  286. plt.autoscale(enable=True, axis='both')
  287. plt.title("MSS Distribution")
  288. plt.xlabel('MSS Value')
  289. plt.ylabel('Number of Packets')
  290. width = 0.5
  291. plt.xlim([0, max(graphx)])
  292. plt.grid(True)
  293. plt.bar(graphx, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  294. out = self.pcap_filepath.replace('.pcap', '_plot-mss' + file_ending)
  295. plt.savefig(out,dpi=500)
  296. return out
  297. # Aidmar
  298. def plot_win(file_ending: str):
  299. plt.gcf().clear()
  300. result = self.stats_db._process_user_defined_query(
  301. "SELECT winSize, SUM(winCount) FROM tcp_syn_win GROUP BY winSize")
  302. graphx, graphy = [], []
  303. for row in result:
  304. graphx.append(row[0])
  305. graphy.append(row[1])
  306. plt.autoscale(enable=True, axis='both')
  307. plt.title("Window Size Distribution")
  308. plt.xlabel('Window Size')
  309. plt.ylabel('Number of Packets')
  310. width = 0.5
  311. plt.xlim([0, max(graphx)])
  312. plt.grid(True)
  313. plt.bar(graphx, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  314. out = self.pcap_filepath.replace('.pcap', '_plot-win' + file_ending)
  315. plt.savefig(out,dpi=500)
  316. return out
  317. # Aidmar
  318. def plot_protocol(file_ending: str):
  319. plt.gcf().clear()
  320. result = self.stats_db._process_user_defined_query(
  321. "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
  322. graphx, graphy = [], []
  323. for row in result:
  324. graphx.append(row[0])
  325. graphy.append(row[1])
  326. plt.autoscale(enable=True, axis='both')
  327. plt.title("Protocols Distribution")
  328. plt.xlabel('Protocols')
  329. plt.ylabel('Number of Packets')
  330. width = 0.5
  331. plt.xlim([0, len(graphx)])
  332. plt.grid(True)
  333. # Protocols' names on x-axis
  334. x = range(0,len(graphx))
  335. my_xticks = graphx
  336. plt.xticks(x, my_xticks)
  337. plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  338. out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
  339. plt.savefig(out,dpi=500)
  340. return out
  341. # Aidmar
  342. def plot_ip_src(file_ending: str):
  343. plt.gcf().clear()
  344. result = self.stats_db._process_user_defined_query(
  345. "SELECT ipAddress, pktsSent FROM ip_statistics")
  346. graphx, graphy = [], []
  347. for row in result:
  348. graphx.append(row[0])
  349. graphy.append(row[1])
  350. plt.autoscale(enable=True, axis='both')
  351. plt.title("Source IP Distribution")
  352. plt.xlabel('Source IP')
  353. plt.ylabel('Number of Packets')
  354. width = 0.5
  355. plt.xlim([0, len(graphx)])
  356. plt.grid(True)
  357. # IPs on x-axis
  358. x = range(0, len(graphx))
  359. my_xticks = graphx
  360. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  361. plt.tight_layout()
  362. plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  363. out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
  364. plt.savefig(out, dpi=500)
  365. return out
  366. # Aidmar
  367. def plot_ip_dst(file_ending: str):
  368. plt.gcf().clear()
  369. result = self.stats_db._process_user_defined_query(
  370. "SELECT ipAddress, pktsReceived FROM ip_statistics")
  371. graphx, graphy = [], []
  372. for row in result:
  373. graphx.append(row[0])
  374. graphy.append(row[1])
  375. plt.autoscale(enable=True, axis='both')
  376. plt.title("Destination IP Distribution")
  377. plt.xlabel('Destination IP')
  378. plt.ylabel('Number of Packets')
  379. width = 0.5
  380. plt.xlim([0, len(graphx)])
  381. plt.grid(True)
  382. # IPs on x-axis
  383. x = range(0, len(graphx))
  384. my_xticks = graphx
  385. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  386. plt.tight_layout()
  387. plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  388. out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
  389. plt.savefig(out, dpi=500)
  390. return out
  391. # Aidmar
  392. def plot_port(file_ending: str):
  393. plt.gcf().clear()
  394. result = self.stats_db._process_user_defined_query(
  395. "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
  396. graphx, graphy = [], []
  397. for row in result:
  398. graphx.append(row[0])
  399. graphy.append(row[1])
  400. plt.autoscale(enable=True, axis='both')
  401. plt.title("Ports Distribution")
  402. plt.xlabel('Ports Numbers')
  403. plt.ylabel('Number of Packets')
  404. width = 0.5
  405. plt.xlim([0, max(graphx)])
  406. plt.grid(True)
  407. plt.bar(graphx, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  408. out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
  409. plt.savefig(out,dpi=500)
  410. return out
  411. # Aidmar
  412. def plot_interval_pktCount(file_ending: str):
  413. plt.gcf().clear()
  414. result = self.stats_db._process_user_defined_query(
  415. "SELECT timestamp, pktsCount FROM interval_statistics ORDER BY timestamp")
  416. graphx, graphy = [], []
  417. for row in result:
  418. graphx.append(row[0])
  419. graphy.append(row[1])
  420. plt.autoscale(enable=True, axis='both')
  421. plt.title("Packet Rate")
  422. plt.xlabel('Timestamp')
  423. plt.ylabel('Number of Packets')
  424. width = 0.5
  425. plt.xlim([0, len(graphx)])
  426. plt.grid(True)
  427. # IPs on x-axis
  428. x = range(0, len(graphx))
  429. my_xticks = graphx
  430. plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
  431. plt.tight_layout()
  432. plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
  433. out = self.pcap_filepath.replace('.pcap', '_plot-interval-pkt-count' + file_ending)
  434. plt.savefig(out, dpi=500)
  435. return out
  436. ttl_out_path = plot_ttl('.' + format)
  437. mss_out_path = plot_mss('.' + format)
  438. win_out_path = plot_win('.' + format)
  439. protocol_out_path = plot_protocol('.' + format)
  440. port_out_path = plot_port('.' + format)
  441. ip_src_out_path = plot_ip_src('.' + format)
  442. ip_dst_out_path = plot_ip_dst('.' + format)
  443. ip_dst_out_path = plot_ip_dst('.' + format)
  444. plot_interval_pktCount = plot_interval_pktCount('.' + format)
  445. print("Saved distributions plots at: %s, %s, %s, %s, %s, %s, %s, %s" %(ttl_out_path,mss_out_path, win_out_path,
  446. protocol_out_path, port_out_path,ip_src_out_path,ip_dst_out_path, plot_interval_pktCount))
  447. """
  448. # Aidmar
  449. graphx_aftr, graphy_aftr = [], []
  450. ttlValue = self.stats_db._process_user_defined_query(
  451. "SELECT ttlValue FROM ip_ttl GROUP BY ttlValue ORDER BY SUM(ttlCount) DESC LIMIT 1")
  452. for row in result:
  453. if(row[0] == 64):
  454. graphy_aftr.append(row[1]+1000)
  455. else:
  456. graphy_aftr.append(row[1])
  457. graphx_aftr.append(row[0])
  458. plt.autoscale(enable=False, axis='both')
  459. plt.title("TTL Distribution")
  460. plt.xlabel('TTL Value')
  461. plt.ylabel('Number of Packets')
  462. width = 0.5
  463. plt.xlim([0, max(graphx_aftr)])
  464. # Aidmar
  465. plt.ylim([0, 11000]) # temp
  466. plt.grid(True)
  467. plt.bar(graphx_aftr, graphy_aftr, width, align='center', linewidth=2, color='red', edgecolor='red')
  468. out = self.pcap_filepath.replace('.pcap', '_plot-ttl_2' + file_ending)
  469. plt.savefig(out)
  470. print(graphy)
  471. print(graphy_aftr)
  472. print("\neuclidean distance: "+str(dist.euclidean(graphy, graphy_aftr)))
  473. print("\ncityblock distance: " + str(dist.cityblock(graphy, graphy_aftr)))
  474. print("\nchebyshev distance: " + str(dist.chebyshev(graphy, graphy_aftr)))
  475. print("\nsqeuclidean distance: " + str(dist.sqeuclidean(graphy, graphy_aftr)))
  476. print("\nhamming distance: " + str(dist.hamming(graphy, graphy_aftr)))
  477. # bhattacharyya test
  478. import math
  479. def mean(hist):
  480. mean = 0.0;
  481. for i in hist:
  482. mean += i;
  483. mean /= len(hist);
  484. return mean;
  485. def bhatta(hist1, hist2):
  486. # calculate mean of hist1
  487. h1_ = mean(hist1);
  488. # calculate mean of hist2
  489. h2_ = mean(hist2);
  490. # calculate score
  491. score = 0;
  492. for i in range(len(hist1)):
  493. score += math.sqrt(hist1[i] * hist2[i]);
  494. # print h1_,h2_,score;
  495. score = math.sqrt(1 - (1 / math.sqrt(h1_ * h2_ * len(hist1) * len(hist1))) * score);
  496. return score;
  497. print("\nbhatta distance: " + str(bhatta(graphy, graphy_aftr)))
  498. """