statistics.cpp 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785
  1. #include <iostream>
  2. #include <fstream>
  3. #include <vector>
  4. #include <math.h>
  5. #include "statistics.h"
  6. #include <sstream>
  7. #include <SQLiteCpp/SQLiteCpp.h>
  8. #include "statistics_db.h"
  9. #include "statistics.h"
  10. #include "utilities.h"
  11. using namespace Tins;
  12. /**
  13. * Checks if there is a payload and increments payloads counter.
  14. * @param pdu_l4 The packet that should be checked if it has a payload or not.
  15. */
  16. void statistics::checkPayload(const PDU *pdu_l4) {
  17. if(this->getDoExtraTests()) {
  18. // pdu_l4: Tarnsport layer 4
  19. int pktSize = pdu_l4->size();
  20. int headerSize = pdu_l4->header_size(); // TCP/UDP header
  21. int payloadSize = pktSize - headerSize;
  22. if (payloadSize > 0)
  23. payloadCount++;
  24. }
  25. }
  26. /**
  27. * Checks the correctness of TCP checksum and increments counter if the checksum was incorrect.
  28. * @param ipAddressSender The source IP.
  29. * @param ipAddressReceiver The destination IP.
  30. * @param tcpPkt The packet to get checked.
  31. */
  32. void statistics::checkTCPChecksum(const IPv4Address ipAddressSender, const IPv4Address ipAddressReceiver, TCP tcpPkt) {
  33. if(this->getDoExtraTests()) {
  34. if(check_tcpChecksum(ipAddressSender, ipAddressReceiver, tcpPkt))
  35. correctTCPChecksumCount++;
  36. else incorrectTCPChecksumCount++;
  37. }
  38. }
  39. /**
  40. * Calculates entropy of the source and destination IPs in a time interval.
  41. * @param intervalStartTimestamp The timstamp where the interval starts.
  42. * @return a vector: contains source IP entropy and destination IP entropy.
  43. */
  44. std::vector<float> statistics::calculateLastIntervalIPsEntropy(std::chrono::microseconds intervalStartTimestamp){
  45. if(this->getDoExtraTests()) {
  46. std::vector<int> IPsSrcPktsCounts;
  47. std::vector<int> IPsDstPktsCounts;
  48. std::vector<float> IPsSrcProb;
  49. std::vector<float> IPsDstProb;
  50. int pktsSent = 0, pktsReceived = 0;
  51. for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
  52. int IPsSrcPktsCount = 0;
  53. for (auto j = i->second.pkts_sent_timestamp.begin(); j != i->second.pkts_sent_timestamp.end(); j++) {
  54. if(*j >= intervalStartTimestamp)
  55. IPsSrcPktsCount++;
  56. }
  57. if(IPsSrcPktsCount != 0) {
  58. IPsSrcPktsCounts.push_back(IPsSrcPktsCount);
  59. pktsSent += IPsSrcPktsCount;
  60. }
  61. int IPsDstPktsCount = 0;
  62. for (auto j = i->second.pkts_received_timestamp.begin(); j != i->second.pkts_received_timestamp.end(); j++) {
  63. if(*j >= intervalStartTimestamp)
  64. IPsDstPktsCount++;
  65. }
  66. if(IPsDstPktsCount != 0) {
  67. IPsDstPktsCounts.push_back(IPsDstPktsCount);
  68. pktsReceived += IPsDstPktsCount;
  69. }
  70. }
  71. for (auto i = IPsSrcPktsCounts.begin(); i != IPsSrcPktsCounts.end(); i++) {
  72. IPsSrcProb.push_back((float) *i / pktsSent);
  73. }
  74. for (auto i = IPsDstPktsCounts.begin(); i != IPsDstPktsCounts.end(); i++) {
  75. IPsDstProb.push_back((float) *i / pktsReceived);
  76. }
  77. // Calculate IP source entropy
  78. float IPsSrcEntropy = 0;
  79. for (unsigned i = 0; i < IPsSrcProb.size(); i++) {
  80. if (IPsSrcProb[i] > 0)
  81. IPsSrcEntropy += -IPsSrcProb[i] * log2(IPsSrcProb[i]);
  82. }
  83. // Calculate IP destination entropy
  84. float IPsDstEntropy = 0;
  85. for (unsigned i = 0; i < IPsDstProb.size(); i++) {
  86. if (IPsDstProb[i] > 0)
  87. IPsDstEntropy += -IPsDstProb[i] * log2(IPsDstProb[i]);
  88. }
  89. std::vector<float> entropies = {IPsSrcEntropy, IPsDstEntropy};
  90. return entropies;
  91. }
  92. else {
  93. return {-1, -1};
  94. }
  95. }
  96. /**
  97. * Calculates the cumulative entropy of the source and destination IPs, i.e., the entropy for packets from the beginning of the pcap file.
  98. * @return a vector: contains the cumulative entropies of source and destination IPs
  99. */
  100. std::vector<float> statistics::calculateIPsCumEntropy(){
  101. if(this->getDoExtraTests()) {
  102. std::vector <IPv4Address> IPs;
  103. std::vector <float> IPsSrcProb;
  104. std::vector <float> IPsDstProb;
  105. for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
  106. IPs.push_back(i->first);
  107. IPsSrcProb.push_back((float)i->second.pkts_sent/packetCount);
  108. IPsDstProb.push_back((float)i->second.pkts_received/packetCount);
  109. }
  110. // Calculate IP source entropy
  111. float IPsSrcEntropy = 0;
  112. for(unsigned i=0; i < IPsSrcProb.size();i++){
  113. if (IPsSrcProb[i] > 0)
  114. IPsSrcEntropy += - IPsSrcProb[i]*log2(IPsSrcProb[i]);
  115. }
  116. // Calculate IP destination entropy
  117. float IPsDstEntropy = 0;
  118. for(unsigned i=0; i < IPsDstProb.size();i++){
  119. if (IPsDstProb[i] > 0)
  120. IPsDstEntropy += - IPsDstProb[i]*log2(IPsDstProb[i]);
  121. }
  122. std::vector<float> entropies = {IPsSrcEntropy, IPsDstEntropy};
  123. return entropies;
  124. }
  125. else {
  126. return {-1, -1};
  127. }
  128. }
  129. /**
  130. * Calculates sending packet rate for each IP in a time interval. Finds min and max packet rate and adds them to ip_statistics map.
  131. * @param intervalStartTimestamp The timstamp where the interval starts.
  132. */
  133. void statistics::calculateIPIntervalPacketRate(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp){
  134. for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
  135. int IPsSrcPktsCount = 0;
  136. for (auto j = i->second.pkts_sent_timestamp.begin(); j != i->second.pkts_sent_timestamp.end(); j++) {
  137. if(*j >= intervalStartTimestamp)
  138. IPsSrcPktsCount++;
  139. }
  140. float interval_pkt_rate = (float) IPsSrcPktsCount * 1000000 / interval.count(); // used 10^6 because interval in microseconds
  141. i->second.interval_pkt_rate.push_back(interval_pkt_rate);
  142. if(interval_pkt_rate > i->second.max_interval_pkt_rate || i->second.max_interval_pkt_rate == 0)
  143. i->second.max_interval_pkt_rate = interval_pkt_rate;
  144. if(interval_pkt_rate < i->second.min_interval_pkt_rate || i->second.min_interval_pkt_rate == 0)
  145. i->second.min_interval_pkt_rate = interval_pkt_rate;
  146. }
  147. }
  148. /**
  149. * Registers statistical data for a time interval.
  150. * @param intervalStartTimestamp The timstamp where the interval starts.
  151. * @param intervalEndTimestamp The timstamp where the interval ends.
  152. * @param previousPacketCount The total number of packets in last interval.
  153. */
  154. void statistics::addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds intervalEndTimestamp){
  155. // Add packet rate for each IP to ip_statistics map
  156. calculateIPIntervalPacketRate(interval, intervalStartTimestamp);
  157. std::vector<float> ipEntopies = calculateLastIntervalIPsEntropy(intervalStartTimestamp);
  158. std::vector<float> ipCumEntopies = calculateIPsCumEntropy();
  159. std::string lastPktTimestamp_s = std::to_string(intervalEndTimestamp.count());
  160. std::string intervalStartTimestamp_s = std::to_string(intervalStartTimestamp.count());
  161. // The intervalStartTimestamp_s is the previous interval lastPktTimestamp_s
  162. interval_statistics[lastPktTimestamp_s].pkts_count = packetCount - intervalCumPktCount;
  163. interval_statistics[lastPktTimestamp_s].kbytes = (float(sumPacketSize - intervalCumSumPktSize) / 1024);
  164. interval_statistics[lastPktTimestamp_s].payload_count = payloadCount - intervalPayloadCount;
  165. interval_statistics[lastPktTimestamp_s].incorrect_tcp_checksum_count = incorrectTCPChecksumCount - intervalIncorrectTCPChecksumCount;
  166. interval_statistics[lastPktTimestamp_s].correct_tcp_checksum_count = correctTCPChecksumCount - intervalCorrectTCPChecksumCount;
  167. interval_statistics[lastPktTimestamp_s].novel_ip_count = ip_statistics.size() - intervalCumNovelIPCount;
  168. interval_statistics[lastPktTimestamp_s].novel_ttl_count = ttl_values.size() - intervalCumNovelTTLCount;
  169. interval_statistics[lastPktTimestamp_s].novel_win_size_count = win_values.size() - intervalCumNovelWinSizeCount;
  170. interval_statistics[lastPktTimestamp_s].novel_tos_count = tos_values.size() - intervalCumNovelToSCount;
  171. interval_statistics[lastPktTimestamp_s].novel_mss_count = mss_values.size() - intervalCumNovelMSSCount;
  172. interval_statistics[lastPktTimestamp_s].novel_port_count = port_values.size() - intervalCumNovelPortCount;
  173. intervalPayloadCount = payloadCount;
  174. intervalIncorrectTCPChecksumCount = incorrectTCPChecksumCount;
  175. intervalCorrectTCPChecksumCount = correctTCPChecksumCount;
  176. intervalCumPktCount = packetCount;
  177. intervalCumSumPktSize = sumPacketSize;
  178. intervalCumNovelIPCount = ip_statistics.size();
  179. intervalCumNovelTTLCount = ttl_values.size();
  180. intervalCumNovelWinSizeCount = win_values.size();
  181. intervalCumNovelToSCount = tos_values.size();
  182. intervalCumNovelMSSCount = mss_values.size();
  183. intervalCumNovelPortCount = port_values.size();
  184. if(ipEntopies.size()>1){
  185. interval_statistics[lastPktTimestamp_s].ip_src_entropy = ipEntopies[0];
  186. interval_statistics[lastPktTimestamp_s].ip_dst_entropy = ipEntopies[1];
  187. }
  188. if(ipCumEntopies.size()>1){
  189. interval_statistics[lastPktTimestamp_s].ip_src_cum_entropy = ipCumEntopies[0];
  190. interval_statistics[lastPktTimestamp_s].ip_dst_cum_entropy = ipCumEntopies[1];
  191. }
  192. }
  193. /**
  194. * Registers statistical data for a sent packet in a given conversation (two IPs, two ports).
  195. * Increments the counter packets_A_B or packets_B_A.
  196. * Adds the timestamp of the packet in pkts_A_B_timestamp or pkts_B_A_timestamp.
  197. * @param ipAddressSender The sender IP address.
  198. * @param sport The source port.
  199. * @param ipAddressReceiver The receiver IP address.
  200. * @param dport The destination port.
  201. * @param timestamp The timestamp of the packet.
  202. */
  203. void statistics::addConvStat(const IPv4Address ipAddressSender, std::uint16_t sport,const IPv4Address ipAddressReceiver, std::uint16_t dport, std::chrono::microseconds timestamp){
  204. conv f1 = {ipAddressReceiver, ipAddressSender, dport, sport};
  205. conv f2 = {ipAddressSender, ipAddressReceiver, sport, dport};
  206. // if already exist A(ipAddressReceiver, dport), B(ipAddressSender, sport) conversation
  207. if (conv_statistics.count(f1)>0){
  208. conv_statistics[f1].pkts_count++;
  209. if(conv_statistics[f1].pkts_count<=3)
  210. conv_statistics[f1].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics[f1].pkts_timestamp.back()));
  211. conv_statistics[f1].pkts_timestamp.push_back(timestamp);
  212. }
  213. // Add new conversation A(ipAddressSender, sport), B(ipAddressReceiver, dport)
  214. else{
  215. conv_statistics[f2].pkts_count++;
  216. if(conv_statistics[f2].pkts_timestamp.size()>0 && conv_statistics[f2].pkts_count<=3 )
  217. conv_statistics[f2].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics[f2].pkts_timestamp.back()));
  218. conv_statistics[f2].pkts_timestamp.push_back(timestamp);
  219. }
  220. }
  221. /**
  222. * Registers statistical data for a sent packet in a given extended conversation (two IPs, two ports, protocol).
  223. * Increments the counter packets_A_B or packets_B_A.
  224. * Adds the timestamp of the packet in pkts_A_B_timestamp or pkts_B_A_timestamp.
  225. * Updates all other statistics of conv_statistics_extended
  226. * @param ipAddressSender The sender IP address.
  227. * @param sport The source port.
  228. * @param ipAddressReceiver The receiver IP address.
  229. * @param dport The destination port.
  230. * @param protocol The used protocol.
  231. * @param timestamp The timestamp of the packet.
  232. */
  233. void statistics::addConvStatExt(const IPv4Address ipAddressSender, std::uint16_t sport, const IPv4Address ipAddressReceiver, std::uint16_t dport, const char * const protocol, std::chrono::microseconds timestamp){
  234. if(this->getDoExtraTests()) {
  235. convWithProt f1 = {ipAddressReceiver, ipAddressSender, dport, sport, protocol};
  236. convWithProt f2 = {ipAddressSender, ipAddressReceiver, sport, dport, protocol};
  237. convWithProt f;
  238. // if there already exists a communication interval for the specified conversation
  239. if (conv_statistics_extended.count(f1) > 0 || conv_statistics_extended.count(f2) > 0){
  240. // find out which direction of conversation is contained in conv_statistics_extended
  241. if (conv_statistics_extended.count(f1) > 0)
  242. f = f1;
  243. else
  244. f = f2;
  245. // increase pkts count and check on delay
  246. conv_statistics_extended[f].pkts_count++;
  247. if (conv_statistics_extended[f].pkts_timestamp.size()>0 && conv_statistics_extended[f].pkts_count<=3)
  248. conv_statistics_extended[f].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics_extended[f].pkts_timestamp.back()));
  249. conv_statistics_extended[f].pkts_timestamp.push_back(timestamp);
  250. // if the time difference has exceeded the threshold, create a new interval with this message
  251. if (timestamp - conv_statistics_extended[f].comm_intervals.back().end > (std::chrono::microseconds) ((unsigned long) COMM_INTERVAL_THRESHOLD)) { // > or >= ?
  252. commInterval new_interval = {timestamp, timestamp, 1};
  253. conv_statistics_extended[f].comm_intervals.push_back(new_interval);
  254. }
  255. // otherwise, set the time of the last interval message to the current timestamp and increase interval packet count by 1
  256. else{
  257. conv_statistics_extended[f].comm_intervals.back().end = timestamp;
  258. conv_statistics_extended[f].comm_intervals.back().pkts_count++;
  259. }
  260. }
  261. // if there does not exist a communication interval for the specified conversation
  262. else{
  263. // add initial interval entry for this conversation
  264. commInterval initial_interval = {timestamp, timestamp, 1};
  265. entry_convStatExt entry;
  266. entry.comm_intervals.push_back(initial_interval);
  267. entry.pkts_count = 1;
  268. entry.pkts_timestamp.push_back(timestamp);
  269. conv_statistics_extended[f2] = entry;
  270. }
  271. }
  272. }
  273. /**
  274. * Aggregate the collected information about all communication intervals within conv_statistics_extended of every conversation.
  275. * Do this by computing the average packet rate per interval and the average time between intervals.
  276. * Also compute average interval duration and total communication duration (i.e. last_msg.time - first_msg.time)
  277. */
  278. void statistics::createCommIntervalStats(){
  279. // iterate over all <convWithProt, entry_convStatExt> pairs
  280. for (auto &cur_elem : conv_statistics_extended) {
  281. entry_convStatExt &entry = cur_elem.second;
  282. std::vector<commInterval> &intervals = entry.comm_intervals;
  283. // if there is only one interval, the time between intervals cannot be computed and is therefore set to 0
  284. if (intervals.size() == 1){
  285. double interval_duration = (double) (intervals[0].end - intervals[0].start).count() / (double) 1e6;
  286. entry.avg_int_pkts_count = (double) intervals[0].pkts_count;
  287. entry.avg_time_between_ints = (double) 0;
  288. entry.avg_interval_time = interval_duration;
  289. }
  290. // If there is more than one interval, compute the specified averages
  291. else if (intervals.size() > 1){
  292. long summed_pkts_count = intervals[0].pkts_count;
  293. std::chrono::microseconds time_between_ints_sum = (std::chrono::microseconds) 0;
  294. std::chrono::microseconds summed_int_duration = intervals[0].end - intervals[0].start;
  295. for (std::size_t i = 1; i < intervals.size(); i++) {
  296. summed_pkts_count += intervals[i].pkts_count;
  297. summed_int_duration += intervals[i].end - intervals[i].start;
  298. time_between_ints_sum += intervals[i].start - intervals[i - 1].end;
  299. }
  300. entry.avg_int_pkts_count = summed_pkts_count / ((double) intervals.size());
  301. entry.avg_time_between_ints = (time_between_ints_sum.count() / (double) (intervals.size() - 1)) / (double) 1e6;
  302. entry.avg_interval_time = (summed_int_duration.count() / (double) intervals.size()) / (double) 1e6;
  303. }
  304. entry.total_comm_duration = (double) (entry.pkts_timestamp.back() - entry.pkts_timestamp.front()).count() / (double) 1e6;
  305. }
  306. }
  307. /**
  308. * Increments the packet counter for the given IP address and MSS value.
  309. * @param ipAddress The IP address whose MSS packet counter should be incremented.
  310. * @param mssValue The MSS value of the packet.
  311. */
  312. void statistics::incrementMSScount(const IPv4Address ipAddress, int mssValue) {
  313. mss_values[mssValue]++;
  314. mss_distribution[{ipAddress, mssValue}]++;
  315. }
  316. /**
  317. * Increments the packet counter for the given IP address and window size.
  318. * @param ipAddress The IP address whose window size packet counter should be incremented.
  319. * @param winSize The window size of the packet.
  320. */
  321. void statistics::incrementWinCount(const IPv4Address ipAddress, int winSize) {
  322. win_values[winSize]++;
  323. win_distribution[{ipAddress, winSize}]++;
  324. }
  325. /**
  326. * Increments the packet counter for the given IP address and TTL value.
  327. * @param ipAddress The IP address whose TTL packet counter should be incremented.
  328. * @param ttlValue The TTL value of the packet.
  329. */
  330. void statistics::incrementTTLcount(const IPv4Address ipAddress, int ttlValue) {
  331. ttl_values[ttlValue]++;
  332. ttl_distribution[{ipAddress, ttlValue}]++;
  333. }
  334. /**
  335. * Increments the packet counter for the given IP address and ToS value.
  336. * @param ipAddress The IP address whose ToS packet counter should be incremented.
  337. * @param tosValue The ToS value of the packet.
  338. */
  339. void statistics::incrementToScount(const IPv4Address ipAddress, int tosValue) {
  340. tos_values[tosValue]++;
  341. tos_distribution[{ipAddress, tosValue}]++;
  342. }
  343. /**
  344. * Increments the protocol counter for the given IP address and protocol.
  345. * @param ipAddress The IP address whose protocol packet counter should be incremented.
  346. * @param protocol The protocol of the packet.
  347. */
  348. void statistics::incrementProtocolCount(const IPv4Address ipAddress, const char * const protocol) {
  349. protocol_distribution[{ipAddress, protocol}].count++;
  350. }
  351. /**
  352. * Returns the number of packets seen for the given IP address and protocol.
  353. * @param ipAddress The IP address whose packet count is wanted.
  354. * @param protocol The protocol whose packet count is wanted.
  355. */
  356. int statistics::getProtocolCount(const IPv4Address ipAddress, const char * const protocol) {
  357. return protocol_distribution[{ipAddress, protocol}].count;
  358. }
  359. /**
  360. * Increases the byte counter for the given IP address and protocol.
  361. * @param ipAddress The IP address whose protocol byte counter should be increased.
  362. * @param protocol The protocol of the packet.
  363. * @param byteSent The packet's size.
  364. */
  365. void statistics::increaseProtocolByteCount(const IPv4Address ipAddress, const char * const protocol, long bytesSent) {
  366. protocol_distribution[{ipAddress, protocol}].byteCount += bytesSent;
  367. }
  368. /**
  369. * Returns the number of bytes seen for the given IP address and protocol.
  370. * @param ipAddress The IP address whose byte count is wanted.
  371. * @param protocol The protocol whose byte count is wanted.
  372. * @return a float: The number of bytes
  373. */
  374. float statistics::getProtocolByteCount(const IPv4Address ipAddress, const char * const protocol) {
  375. return protocol_distribution[{ipAddress, protocol}].byteCount;
  376. }
  377. /**
  378. * Increments the packet counter for
  379. * - the given sender IP address with outgoing port and
  380. * - the given receiver IP address with incoming port.
  381. * @param ipAddressSender The IP address of the packet sender.
  382. * @param outgoingPort The port used by the sender.
  383. * @param ipAddressReceiver The IP address of the packet receiver.
  384. * @param incomingPort The port used by the receiver.
  385. */
  386. void statistics::incrementPortCount(const IPv4Address ipAddressSender, std::uint16_t outgoingPort, const IPv4Address ipAddressReceiver,
  387. std::uint16_t incomingPort, const char * const protocol) {
  388. port_values[outgoingPort]++;
  389. port_values[incomingPort]++;
  390. ip_ports[{ipAddressSender, outgoingPort, DIR_OUT, protocol}].count++;
  391. ip_ports[{ipAddressReceiver, incomingPort, DIR_IN, protocol}].count++;
  392. }
  393. /**
  394. * Increases the packet byte counter for
  395. * - the given sender IP address with outgoing port and
  396. * - the given receiver IP address with incoming port.
  397. * @param ipAddressSender The IP address of the packet sender.
  398. * @param outgoingPort The port used by the sender.
  399. * @param ipAddressReceiver The IP address of the packet receiver.
  400. * @param incomingPort The port used by the receiver.
  401. * @param byteSent The packet's size.
  402. */
  403. void statistics::increasePortByteCount(const IPv4Address ipAddressSender, std::uint16_t outgoingPort, const IPv4Address ipAddressReceiver,
  404. std::uint16_t incomingPort, long bytesSent, const char * const protocol) {
  405. ip_ports[{ipAddressSender, outgoingPort, DIR_OUT, protocol}].byteCount += bytesSent;
  406. ip_ports[{ipAddressReceiver, incomingPort, DIR_IN, protocol}].byteCount += bytesSent;
  407. }
  408. /**
  409. * Increments the packet counter for
  410. * - the given sender MAC address and
  411. * - the given receiver MAC address.
  412. * @param srcMac The MAC address of the packet sender.
  413. * @param dstMac The MAC address of the packet receiver.
  414. * @param typeNumber The payload type number of the packet.
  415. */
  416. void statistics::incrementUnrecognizedPDUCount(const std::string &srcMac, const std::string &dstMac, uint32_t typeNumber,
  417. const std::string &timestamp) {
  418. unrecognized_PDUs[{srcMac, dstMac, typeNumber}].count++;
  419. unrecognized_PDUs[{srcMac, dstMac, typeNumber}].timestamp_last_occurrence = timestamp;
  420. }
  421. /**
  422. * Creates a new statistics object.
  423. */
  424. statistics::statistics(void) {
  425. }
  426. /**
  427. * Stores the assignment IP address -> MAC address.
  428. * @param ipAddress The IP address belonging to the given MAC address.
  429. * @param macAddress The MAC address belonging to the given IP address.
  430. */
  431. void statistics::assignMacAddress(const IPv4Address ipAddress, const std::string &macAddress) {
  432. ip_mac_mapping[ipAddress] = macAddress;
  433. }
  434. /**
  435. * Registers statistical data for a sent packet. Increments the counter packets_sent for the sender and
  436. * packets_received for the receiver. Adds the bytes as kbytes_sent (sender) and kybtes_received (receiver).
  437. * @param ipAddressSender The IP address of the packet sender.
  438. * @param ipAddressReceiver The IP address of the packet receiver.
  439. * @param bytesSent The packet's size.
  440. */
  441. void statistics::addIpStat_packetSent(const IPv4Address ipAddressSender, const IPv4Address ipAddressReceiver, long bytesSent, std::chrono::microseconds timestamp) {
  442. // Adding IP as a sender for first time
  443. if(ip_statistics[ipAddressSender].pkts_sent==0){
  444. // Add the IP class
  445. ip_statistics[ipAddressSender].ip_class = getIPv4Class(ipAddressSender);
  446. }
  447. // Adding IP as a receiver for first time
  448. if(ip_statistics[ipAddressReceiver].pkts_received==0){
  449. // Add the IP class
  450. ip_statistics[ipAddressReceiver].ip_class = getIPv4Class(ipAddressReceiver);
  451. }
  452. // Update stats for packet sender
  453. ip_statistics[ipAddressSender].kbytes_sent += (float(bytesSent) / 1024);
  454. ip_statistics[ipAddressSender].pkts_sent++;
  455. ip_statistics[ipAddressSender].pkts_sent_timestamp.push_back(timestamp);
  456. // Update stats for packet receiver
  457. ip_statistics[ipAddressReceiver].kbytes_received += (float(bytesSent) / 1024);
  458. ip_statistics[ipAddressReceiver].pkts_received++;
  459. ip_statistics[ipAddressReceiver].pkts_received_timestamp.push_back(timestamp);
  460. if(this->getDoExtraTests()) {
  461. // Increment Degrees for sender and receiver, if Sender sends its first packet to this receiver
  462. std::unordered_set<IPv4Address>::const_iterator found_receiver = contacted_ips[ipAddressSender].find(ipAddressReceiver);
  463. if(found_receiver == contacted_ips[ipAddressSender].end()){
  464. // Receiver is NOT contained in the List of IPs, that the Sender has contacted, therefore this is the first packet in this direction
  465. ip_statistics[ipAddressSender].out_degree++;
  466. ip_statistics[ipAddressReceiver].in_degree++;
  467. // Increment overall_degree only if this is the first packet for the connection (both directions)
  468. // Therefore check, whether Receiver has contacted Sender before
  469. std::unordered_set<IPv4Address>::const_iterator sender_contacted = contacted_ips[ipAddressReceiver].find(ipAddressSender);
  470. if(sender_contacted == contacted_ips[ipAddressReceiver].end()){
  471. ip_statistics[ipAddressSender].overall_degree++;
  472. ip_statistics[ipAddressReceiver].overall_degree++;
  473. }
  474. contacted_ips[ipAddressSender].insert(ipAddressReceiver);
  475. }
  476. }
  477. }
  478. /**
  479. * Setter for the timestamp_firstPacket field.
  480. * @param ts The timestamp of the first packet in the PCAP file.
  481. */
  482. void statistics::setTimestampFirstPacket(Tins::Timestamp ts) {
  483. timestamp_firstPacket = ts;
  484. }
  485. /**
  486. * Setter for the timestamp_lastPacket field.
  487. * @param ts The timestamp of the last packet in the PCAP file.
  488. */
  489. void statistics::setTimestampLastPacket(Tins::Timestamp ts) {
  490. timestamp_lastPacket = ts;
  491. }
  492. /**
  493. * Getter for the timestamp_firstPacket field.
  494. */
  495. Tins::Timestamp statistics::getTimestampFirstPacket() {
  496. return timestamp_firstPacket;
  497. }
  498. /**
  499. * Getter for the timestamp_lastPacket field.
  500. */
  501. Tins::Timestamp statistics::getTimestampLastPacket() {
  502. return timestamp_lastPacket;
  503. }
  504. /**
  505. * Getter for the packetCount field.
  506. */
  507. int statistics::getPacketCount() {
  508. return packetCount;
  509. }
  510. /**
  511. * Getter for the sumPacketSize field.
  512. */
  513. int statistics::getSumPacketSize() {
  514. return sumPacketSize;
  515. }
  516. /**
  517. * Returns the average packet size.
  518. * @return a float indicating the average packet size in kbytes.
  519. */
  520. float statistics::getAvgPacketSize() const {
  521. // AvgPktSize = (Sum of all packet sizes / #Packets)
  522. return (sumPacketSize / packetCount) / 1024;
  523. }
  524. /**
  525. * Adds the size of a packet (to be used to calculate the avg. packet size).
  526. * @param packetSize The size of the current packet in bytes.
  527. */
  528. void statistics::addPacketSize(uint32_t packetSize) {
  529. sumPacketSize += ((float) packetSize);
  530. }
  531. /**
  532. * Setter for the doExtraTests field.
  533. */
  534. void statistics::setDoExtraTests(bool var) {
  535. doExtraTests = var;
  536. }
  537. /**
  538. * Getter for the doExtraTests field.
  539. */
  540. bool statistics::getDoExtraTests() {
  541. return doExtraTests;
  542. }
  543. /**
  544. * Calculates the capture duration.
  545. * @return a formatted string HH:MM:SS.mmmmmm with
  546. * HH: hour, MM: minute, SS: second, mmmmmm: microseconds
  547. */
  548. std::string statistics::getCaptureDurationTimestamp() const {
  549. // Calculate duration
  550. timeval fp, lp, d;
  551. fp.tv_sec = timestamp_firstPacket.seconds();
  552. fp.tv_usec = timestamp_firstPacket.microseconds();
  553. lp.tv_sec = timestamp_lastPacket.seconds();
  554. lp.tv_usec = timestamp_lastPacket.microseconds();
  555. timersub(&lp, &fp, &d);
  556. long int hour = d.tv_sec / 3600;
  557. long int remainder = (d.tv_sec - hour * 3600);
  558. long int minute = remainder / 60;
  559. long int second = (remainder - minute * 60) % 60;
  560. long int microseconds = d.tv_usec;
  561. // Build desired output format: YYYY-mm-dd hh:mm:ss
  562. char out[64];
  563. sprintf(out, "%02ld:%02ld:%02ld.%06ld ", hour, minute, second, microseconds);
  564. return std::string(out);
  565. }
  566. /**
  567. * Calculates the capture duration.
  568. * @return a formatted string SS.mmmmmm with
  569. * S: seconds (UNIX time), mmmmmm: microseconds
  570. */
  571. float statistics::getCaptureDurationSeconds() const {
  572. timeval fp, lp, d;
  573. fp.tv_sec = timestamp_firstPacket.seconds();
  574. fp.tv_usec = timestamp_firstPacket.microseconds();
  575. lp.tv_sec = timestamp_lastPacket.seconds();
  576. lp.tv_usec = timestamp_lastPacket.microseconds();
  577. timersub(&lp, &fp, &d);
  578. char buf[64];
  579. snprintf(buf, sizeof(buf), "%u.%06u", static_cast<uint>(d.tv_sec), static_cast<uint>(d.tv_usec));
  580. return std::stof(std::string(buf));
  581. }
  582. /**
  583. * Creates a timestamp based on a time_t seconds (UNIX time format) and microseconds.
  584. * @param seconds
  585. * @param microseconds
  586. * @return a formatted string Y-m-d H:M:S.m with
  587. * Y: year, m: month, d: day, H: hour, M: minute, S: second, m: microseconds
  588. */
  589. std::string statistics::getFormattedTimestamp(time_t seconds, suseconds_t microseconds) const {
  590. timeval tv;
  591. tv.tv_sec = seconds;
  592. tv.tv_usec = microseconds;
  593. char tmbuf[64], buf[64];
  594. auto nowtm = localtime(&(tv.tv_sec));
  595. strftime(tmbuf, sizeof(tmbuf), "%Y-%m-%d %H:%M:%S", nowtm);
  596. snprintf(buf, sizeof(buf), "%s.%06u", tmbuf, static_cast<uint>(tv.tv_usec));
  597. return std::string(buf);
  598. }
  599. /**
  600. * Calculates the statistics for a given IP address.
  601. * @param ipAddress The IP address whose statistics should be calculated.
  602. * @return a ip_stats struct containing statistical data derived by the statistical data collected.
  603. */
  604. ip_stats statistics::getStatsForIP(const IPv4Address ipAddress) {
  605. float duration = getCaptureDurationSeconds();
  606. entry_ipStat ipStatEntry = ip_statistics[ipAddress];
  607. ip_stats s;
  608. s.bandwidthKBitsIn = (ipStatEntry.kbytes_received / duration) * 8;
  609. s.bandwidthKBitsOut = (ipStatEntry.kbytes_sent / duration) * 8;
  610. s.packetPerSecondIn = (ipStatEntry.pkts_received / duration);
  611. s.packetPerSecondOut = (ipStatEntry.pkts_sent / duration);
  612. s.AvgPacketSizeSent = (ipStatEntry.kbytes_sent / ipStatEntry.pkts_sent);
  613. s.AvgPacketSizeRecv = (ipStatEntry.kbytes_received / ipStatEntry.pkts_received);
  614. return s;
  615. }
  616. /**
  617. * Increments the packet counter.
  618. */
  619. void statistics::incrementPacketCount() {
  620. packetCount++;
  621. }
  622. /**
  623. * Prints the statistics of the PCAP and IP specific statistics for the given IP address.
  624. * @param ipAddress The IP address whose statistics should be printed. Can be empty "" to print only general file statistics.
  625. */
  626. void statistics::printStats(const std::string &ipAddress) {
  627. std::stringstream ss;
  628. ss << std::endl;
  629. ss << "Capture duration: " << getCaptureDurationSeconds() << " seconds" << std::endl;
  630. ss << "Capture duration (HH:MM:SS.mmmmmm): " << getCaptureDurationTimestamp() << std::endl;
  631. ss << "#Packets: " << packetCount << std::endl;
  632. ss << std::endl;
  633. // Print IP address specific statistics only if IP address was given
  634. if (ipAddress != "") {
  635. entry_ipStat e = ip_statistics[ipAddress];
  636. ss << "\n----- STATS FOR IP ADDRESS [" << ipAddress << "] -------" << std::endl;
  637. ss << std::endl << "KBytes sent: " << e.kbytes_sent << std::endl;
  638. ss << "KBytes received: " << e.kbytes_received << std::endl;
  639. ss << "Packets sent: " << e.pkts_sent << std::endl;
  640. ss << "Packets received: " << e.pkts_received << "\n\n";
  641. ip_stats is = getStatsForIP(ipAddress);
  642. ss << "Bandwidth IN: " << is.bandwidthKBitsIn << " kbit/s" << std::endl;
  643. ss << "Bandwidth OUT: " << is.bandwidthKBitsOut << " kbit/s" << std::endl;
  644. ss << "Packets per second IN: " << is.packetPerSecondIn << std::endl;
  645. ss << "Packets per second OUT: " << is.packetPerSecondOut << std::endl;
  646. ss << "Avg Packet Size Sent: " << is.AvgPacketSizeSent << " kbytes" << std::endl;
  647. ss << "Avg Packet Size Received: " << is.AvgPacketSizeRecv << " kbytes" << std::endl;
  648. }
  649. std::cout << ss.str();
  650. }
  651. /**
  652. * Derives general PCAP file statistics from the collected statistical data and
  653. * writes all data into a SQLite database, located at database_path.
  654. * @param database_path The path of the SQLite database file ending with .sqlite3.
  655. */
  656. void statistics::writeToDatabase(std::string database_path) {
  657. // Generate general file statistics
  658. float duration = getCaptureDurationSeconds();
  659. long sumPacketsSent = 0, senderCountIP = 0;
  660. float sumBandwidthIn = 0.0, sumBandwidthOut = 0.0;
  661. for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
  662. sumPacketsSent += i->second.pkts_sent;
  663. // Consumed bandwith (bytes) for sending packets
  664. sumBandwidthIn += (i->second.kbytes_received / duration);
  665. sumBandwidthOut += (i->second.kbytes_sent / duration);
  666. senderCountIP++;
  667. }
  668. float avgPacketRate = (packetCount / duration);
  669. long avgPacketSize = getAvgPacketSize();
  670. if(senderCountIP>0) {
  671. long avgPacketsSentPerHost = (sumPacketsSent / senderCountIP);
  672. float avgBandwidthInKBits = (sumBandwidthIn / senderCountIP) * 8;
  673. float avgBandwidthOutInKBits = (sumBandwidthOut / senderCountIP) * 8;
  674. // Create database and write information
  675. statistics_db db(database_path);
  676. db.writeStatisticsFile(packetCount, getCaptureDurationSeconds(),
  677. getFormattedTimestamp(timestamp_firstPacket.seconds(), timestamp_firstPacket.microseconds()),
  678. getFormattedTimestamp(timestamp_lastPacket.seconds(), timestamp_lastPacket.microseconds()),
  679. avgPacketRate, avgPacketSize, avgPacketsSentPerHost, avgBandwidthInKBits,
  680. avgBandwidthOutInKBits, doExtraTests);
  681. db.writeStatisticsIP(ip_statistics);
  682. db.writeStatisticsTTL(ttl_distribution);
  683. db.writeStatisticsIpMac(ip_mac_mapping);
  684. db.writeStatisticsDegree(ip_statistics);
  685. db.writeStatisticsPorts(ip_ports);
  686. db.writeStatisticsProtocols(protocol_distribution);
  687. db.writeStatisticsMSS(mss_distribution);
  688. db.writeStatisticsToS(tos_distribution);
  689. db.writeStatisticsWin(win_distribution);
  690. db.writeStatisticsConv(conv_statistics);
  691. db.writeStatisticsConvExt(conv_statistics_extended);
  692. db.writeStatisticsInterval(interval_statistics);
  693. db.writeDbVersion();
  694. db.writeStatisticsUnrecognizedPDUs(unrecognized_PDUs);
  695. }
  696. else {
  697. // Tinslib failed to recognize the types of the packets in the input PCAP
  698. std::cout<<"ERROR: Statistics could not be collected from the input PCAP!"<<"\n";
  699. return;
  700. }
  701. }