Merge branch 'db_versioning' of stefan.schmidt/ID2T-toolkit into master

code/ID2TLib/Statistics.py

@@ -54,7 +54,7 @@ class Statistics:
             print("Flag -r/--recalculate found. Recalculating statistics.")
 
         # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
-        if (not self.stats_db.get_db_exists()) or flag_recalculate_stats:
+        if (not self.stats_db.get_db_exists()) or flag_recalculate_stats or self.stats_db.get_db_outdated():
             self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests))
             self.pcap_proc.collect_statistics()
             self.pcap_proc.write_to_database(self.path_db)

code/ID2TLib/StatsDatabase.py

@@ -3,6 +3,7 @@ import re
 import sqlite3
 import sys
 from random import randint
+import ID2TLib.libpcapreader as pr
 
 
 def dict_gen(curs: sqlite3.Cursor):
@@ -32,7 +33,10 @@ class StatsDatabase:
 
         # If DB not existing, create a new DB scheme
         if self.existing_db:
-            print('Located statistics database at: ', db_path)
+            if self.get_db_outdated():
+                print('Statistics database outdated. Recreating database at: ', db_path)
+            else:
+                print('Located statistics database at: ', db_path)
         else:
             print('Statistics database not found. Creating new database at: ', db_path)
 
@@ -61,6 +65,15 @@ class StatsDatabase:
         """
         return self.existing_db
 
+    def get_db_outdated(self):
+        """
+        Retrieves the database version from the database and compares it to the version
+        it should have to check whether the database is outdated and needs to be recreated.
+        :return: True if the versions match, otherwise False
+        """
+        self.cursor.execute('PRAGMA user_version;')
+        return self.cursor.fetchall()[0][0] != pr.pcap_processor.get_db_version()
+
     @staticmethod
     def _get_selector_keywords():
         """

code_boost/src/cxx/pcap_processor.cpp

@@ -211,6 +211,7 @@ void pcap_processor::process_packets(const Packet &pkt) {
 
         // Protocol distribution
         stats.incrementProtocolCount(ipAddressSender, "IPv4");
+        stats.increaseProtocolByteCount(ipAddressSender, "IPv4", sizeCurrentPacket);
 
         // Assign IP Address to MAC Address
         stats.assignMacAddress(ipAddressSender, macAddressSender);
@@ -230,6 +231,7 @@ void pcap_processor::process_packets(const Packet &pkt) {
 
         // Protocol distribution
         stats.incrementProtocolCount(ipAddressSender, "IPv6");
+        stats.increaseProtocolByteCount(ipAddressSender, "IPv6", sizeCurrentPacket);
 
         // Assign IP Address to MAC Address
         stats.assignMacAddress(ipAddressSender, macAddressSender);
@@ -257,7 +259,8 @@ void pcap_processor::process_packets(const Packet &pkt) {
             stats.checkTCPChecksum(ipAddressSender, ipAddressReceiver, tcpPkt);
           }
 
-            stats.incrementProtocolCount(ipAddressSender, "TCP");                        
+            stats.incrementProtocolCount(ipAddressSender, "TCP");
+            stats.increaseProtocolByteCount(ipAddressSender, "TCP", sizeCurrentPacket);
 
             // Conversation statistics
             stats.addConvStat(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp());
@@ -275,17 +278,22 @@ void pcap_processor::process_packets(const Packet &pkt) {
                 // Ignore MSS if option not set
             }
             stats.incrementPortCount(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport());
+            stats.increasePortByteCount(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), sizeCurrentPacket);
             
           // UDP Packet
         } else if (p == PDU::PDUType::UDP) {
             const UDP udpPkt = (const UDP &) *pdu_l4;
-            stats.incrementProtocolCount(ipAddressSender, "UDP");            
-            stats.incrementPortCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport());                        
+            stats.incrementProtocolCount(ipAddressSender, "UDP");
+            stats.increaseProtocolByteCount(ipAddressSender, "UDP", sizeCurrentPacket);
+            stats.incrementPortCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport());
+            stats.increasePortByteCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport(), sizeCurrentPacket);
           
         } else if (p == PDU::PDUType::ICMP) {
             stats.incrementProtocolCount(ipAddressSender, "ICMP");
+            stats.increaseProtocolByteCount(ipAddressSender, "ICMP", sizeCurrentPacket);
         } else if (p == PDU::PDUType::ICMPv6) {
             stats.incrementProtocolCount(ipAddressSender, "ICMPv6");
+            stats.increaseProtocolByteCount(ipAddressSender, "ICMPv6", sizeCurrentPacket);
         }
     }
 }
@@ -347,5 +355,6 @@ BOOST_PYTHON_MODULE (libpcapreader) {
             .def("merge_pcaps", &pcap_processor::merge_pcaps)
             .def("collect_statistics", &pcap_processor::collect_statistics)
             .def("get_timestamp_mu_sec", &pcap_processor::get_timestamp_mu_sec)
-            .def("write_to_database", &pcap_processor::write_to_database);
+            .def("write_to_database", &pcap_processor::write_to_database)
+            .def("get_db_version", &pcap_processor::get_db_version).staticmethod("get_db_version");
 }

code_boost/src/cxx/pcap_processor.h

@@ -13,6 +13,7 @@
 #include <sys/stat.h>
 #include <unordered_map>
 #include "statistics.h"
+#include "statistics_db.h"
 
 using namespace Tins;
 
@@ -44,6 +45,8 @@ public:
     void collect_statistics();
 
     void write_to_database(std::string database_path);
+
+    static int get_db_version() { return statistics_db::DB_VERSION; };
 };
 
 

code_boost/src/cxx/statistics.cpp

@@ -293,17 +293,37 @@ void statistics::incrementToScount(std::string ipAddress, int tosValue) {
  * @param protocol The protocol of the packet.
  */
 void statistics::incrementProtocolCount(std::string ipAddress, std::string protocol) {
-    protocol_distribution[{ipAddress, protocol}]++;
+    protocol_distribution[{ipAddress, protocol}].count++;
 }
 
 /**
  * Returns the number of packets seen for the given IP address and protocol.
  * @param ipAddress The IP address whose packet count is wanted.
  * @param protocol The protocol whose packet count is wanted.
- * @return an integer: the number of packets
+ * @return an integer: The number of packets
  */
 int statistics::getProtocolCount(std::string ipAddress, std::string protocol) {
-    return protocol_distribution[{ipAddress, protocol}];
+    return protocol_distribution[{ipAddress, protocol}].count;
+}
+
+/**
+ * Increases the byte counter for the given IP address and protocol.
+ * @param ipAddress The IP address whose protocol byte counter should be increased.
+ * @param protocol The protocol of the packet.
+ * @param byteSent The packet's size.
+ */
+void statistics::increaseProtocolByteCount(std::string ipAddress, std::string protocol, long bytesSent) {
+    protocol_distribution[{ipAddress, protocol}].byteCount += bytesSent;
+}
+
+/**
+ * Returns the number of bytes seen for the given IP address and protocol.
+ * @param ipAddress The IP address whose byte count is wanted.
+ * @param protocol The protocol whose byte count is wanted.
+ * @return a float: The number of bytes
+ */
+float statistics::getProtocolByteCount(std::string ipAddress, std::string protocol) {
+    return protocol_distribution[{ipAddress, protocol}].byteCount;
 }
 
 /**
@@ -319,9 +339,24 @@ void statistics::incrementPortCount(std::string ipAddressSender, int outgoingPor
                                     int incomingPort) {
     port_values[outgoingPort]++;
     port_values[incomingPort]++;
-    ip_ports[{ipAddressSender, "out", outgoingPort}]++;
-    ip_ports[{ipAddressReceiver, "in", incomingPort}]++;
+    ip_ports[{ipAddressSender, "out", outgoingPort}].count++;
+    ip_ports[{ipAddressReceiver, "in", incomingPort}].count++;
+}
 
+/**
+ * Increases the packet byte counter for
+ * - the given sender IP address with outgoing port and
+ * - the given receiver IP address with incoming port.
+ * @param ipAddressSender The IP address of the packet sender.
+ * @param outgoingPort The port used by the sender.
+ * @param ipAddressReceiver The IP address of the packet receiver.
+ * @param incomingPort The port used by the receiver.
+ * @param byteSent The packet's size.
+ */
+void statistics::increasePortByteCount(std::string ipAddressSender, int outgoingPort, std::string ipAddressReceiver,
+                                       int incomingPort, long bytesSent) {
+    ip_ports[{ipAddressSender, "out", outgoingPort}].byteCount += bytesSent;
+    ip_ports[{ipAddressReceiver, "in", incomingPort}].byteCount += bytesSent;
 }
 
 /**
@@ -602,6 +637,7 @@ void statistics::writeToDatabase(std::string database_path) {
         db.writeStatisticsWin(win_distribution);
         db.writeStatisticsConv(conv_statistics);
         db.writeStatisticsInterval(interval_statistics);
+        db.writeDbVersion();
     }
     else {
         // Tinslib failed to recognize the types of the packets in the input PCAP
@@ -609,42 +645,3 @@ void statistics::writeToDatabase(std::string database_path) {
         return;
     }
 }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

code_boost/src/cxx/statistics.h

@@ -169,6 +169,26 @@ struct entry_ipStat {
     }
 };
 
+/*
+ * Struct used to represent:
+ * - Number of transmitted packets
+ * - Number of transmitted bytes
+ */
+struct entry_portStat {
+    int count;
+    float byteCount;
+};
+
+/*
+ * Struct used to represent:
+ * - Number of times the protocol is seen
+ * - Amount of bytes transmitted with this protocol
+ */
+struct entry_protocolStat {
+    int count;
+    float byteCount;
+};
+
 /*
  * Struct used to represent interval statistics:
  * - # packets
@@ -391,11 +411,18 @@ public:
 
     void incrementProtocolCount(std::string ipAddress, std::string protocol);
 
+    void increaseProtocolByteCount(std::string ipAddress, std::string protocol, long bytesSent);
+
     void incrementPortCount(std::string ipAddressSender, int outgoingPort, std::string ipAddressReceiver,
                             int incomingPort);
 
+    void increasePortByteCount(std::string ipAddressSender, int outgoingPort, std::string ipAddressReceiver,
+                               int incomingPort, long bytesSent);
+
     int getProtocolCount(std::string ipAddress, std::string protocol);
 
+    float getProtocolByteCount(std::string ipAddress, std::string protocol);
+
     void setTimestampFirstPacket(Tins::Timestamp ts);
 
     void setTimestampLastPacket(Tins::Timestamp ts);
@@ -507,14 +534,14 @@ private:
     // {Port, count}
     std::unordered_map<int, int> port_values;
 
-    // {IP Address, Protocol, count}
-    std::unordered_map<ipAddress_protocol, int> protocol_distribution;
+    // {IP Address, Protocol,  #count, #Data transmitted in bytes}
+    std::unordered_map<ipAddress_protocol, entry_protocolStat> protocol_distribution;
 
     // {IP Address,  #received packets, #sent packets, Data received in kbytes, Data sent in kbytes}
     std::unordered_map<std::string, entry_ipStat> ip_statistics;
 
-    // {IP Address, in_out, Port Number, count}
-    std::unordered_map<ipAddress_inOut_port, int> ip_ports;
+    // {IP Address, in_out, Port Number,  #count, #Data transmitted in bytes}
+    std::unordered_map<ipAddress_inOut_port, entry_portStat> ip_ports;
 
     // {IP Address, MAC Address}
     std::unordered_map<std::string, std::string> ip_mac_mapping;

code_boost/src/cxx/statistics_db.cpp

@@ -181,7 +181,7 @@ void statistics_db::writeStatisticsWin(std::unordered_map<ipAddress_win, int> wi
  * Writes the protocol distribution into the database.
  * @param protocolDistribution The protocol distribution from class statistics.
  */
-void statistics_db::writeStatisticsProtocols(std::unordered_map<ipAddress_protocol, int> protocolDistribution) {
+void statistics_db::writeStatisticsProtocols(std::unordered_map<ipAddress_protocol, entry_protocolStat> protocolDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_protocols");
         SQLite::Transaction transaction(*db);
@@ -189,14 +189,16 @@ void statistics_db::writeStatisticsProtocols(std::unordered_map<ipAddress_protoc
                 "ipAddress TEXT,"
                 "protocolName TEXT COLLATE NOCASE,"
                 "protocolCount INTEGER,"
+                "byteCount REAL,"
                 "PRIMARY KEY(ipAddress,protocolName));";
         db->exec(createTable);
-        SQLite::Statement query(*db, "INSERT INTO ip_protocols VALUES (?, ?, ?)");
+        SQLite::Statement query(*db, "INSERT INTO ip_protocols VALUES (?, ?, ?, ?)");
         for (auto it = protocolDistribution.begin(); it != protocolDistribution.end(); ++it) {
             ipAddress_protocol e = it->first;
             query.bind(1, e.ipAddress);
             query.bind(2, e.protocol);
-            query.bind(3, it->second);
+            query.bind(3, it->second.count);
+            query.bind(4, it->second.byteCount);
             query.exec();
             query.reset();
         }
@@ -211,7 +213,7 @@ void statistics_db::writeStatisticsProtocols(std::unordered_map<ipAddress_protoc
  * Writes the port statistics into the database.
  * @param portsStatistics The ports statistics from class statistics.
  */
-void statistics_db::writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port, int> portsStatistics) {
+void statistics_db::writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port, entry_portStat> portsStatistics) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_ports");
         SQLite::Transaction transaction(*db);
@@ -220,15 +222,17 @@ void statistics_db::writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port
                 "portDirection TEXT COLLATE NOCASE,"
                 "portNumber INTEGER,"
                 "portCount INTEGER,"
+                "byteCount REAL,"
                 "PRIMARY KEY(ipAddress,portDirection,portNumber));";
         db->exec(createTable);
-        SQLite::Statement query(*db, "INSERT INTO ip_ports VALUES (?, ?, ?, ?)");
+        SQLite::Statement query(*db, "INSERT INTO ip_ports VALUES (?, ?, ?, ?, ?)");
         for (auto it = portsStatistics.begin(); it != portsStatistics.end(); ++it) {
             ipAddress_inOut_port e = it->first;
             query.bind(1, e.ipAddress);
             query.bind(2, e.trafficDirection);
             query.bind(3, e.portNumber);
-            query.bind(4, it->second);
+            query.bind(4, it->second.count);
+            query.bind(5, it->second.byteCount);
             query.exec();
             query.reset();
         }
@@ -438,3 +442,14 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
     }
 }
 
+void statistics_db::writeDbVersion(){
+	try {
+		SQLite::Transaction transaction(*db);
+		SQLite::Statement query(*db, std::string("PRAGMA user_version = ") + std::to_string(DB_VERSION) + ";");
+		query.exec();
+		transaction.commit();
+	}
+	catch (std::exception &e) {
+        std::cout << "Exception in statistics_db: " << e.what() << std::endl;
+    }
+}

code_boost/src/cxx/statistics_db.h

@@ -18,6 +18,11 @@ public:
      */
     statistics_db(std::string database_path);
 
+    /*
+     * Database version: Increment number on every change in the C++ code!
+     */
+    static const int DB_VERSION = 2;
+
     /*
      * Methods for writing values into database
      */
@@ -31,9 +36,9 @@ public:
 
     void writeStatisticsWin(std::unordered_map<ipAddress_win, int> winDistribution);
 
-    void writeStatisticsProtocols(std::unordered_map<ipAddress_protocol, int> protocolDistribution);
+    void writeStatisticsProtocols(std::unordered_map<ipAddress_protocol, entry_protocolStat> protocolDistribution);
 
-    void writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port, int> portsStatistics);
+    void writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port, entry_portStat> portsStatistics);
 
     void writeStatisticsIpMac(std::unordered_map<std::string, std::string> IpMacStatistics);
 
@@ -45,6 +50,8 @@ public:
 
     void writeStatisticsInterval(std::unordered_map<std::string, entry_intervalStat> intervalStatistics);
 
+    void writeDbVersion();
+
 private:
     // Pointer to the SQLite database
     std::unique_ptr<SQLite::Database> db;