Create a new statistics table containing information on stateless and statefull conversation

Note that since C++ code is changed, ID2T needs to be rebuilt.

code/ID2TLib/Statistics.py

@@ -1031,7 +1031,8 @@ class Statistics:
         def plot_packets_per_connection(file_ending: str):
             plt.gcf().clear()
             result = self.stats_db._process_user_defined_query(
-                "SELECT ipAddressA, portA, ipAddressB, portB, pktsCount FROM conv_statistics")
+                "SELECT ipAddressA, portA, ipAddressB, portB, pktsCount FROM conv_statistics_stateless")
+
             if (result):
                 graphy, graphx = [], []
                 # plot data in descending order

code_boost/src/cxx/pcap_processor.cpp

@@ -253,6 +253,7 @@ void pcap_processor::process_packets(const Packet &pkt) {
 
             // Conversation statistics
             stats.addConvStat(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp());
+            stats.addConvStatStateless(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp()); 
 
             // Window Size distribution
             int win = tcpPkt.window();
@@ -273,7 +274,7 @@ void pcap_processor::process_packets(const Packet &pkt) {
             const UDP udpPkt = (const UDP &) *pdu_l4;
             stats.incrementProtocolCount(ipAddressSender, "UDP");   
             // Conversation statistics
-            stats.addConvStat(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport(), pkt.timestamp());           
+            stats.addConvStatStateless(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport(), pkt.timestamp());           
             stats.incrementPortCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport());             
           
         } else if (p == PDU::PDUType::ICMP) {

code_boost/src/cxx/statistics.cpp

@@ -247,6 +247,37 @@ void statistics::addConvStat(std::string ipAddressSender,int sport,std::string i
     }
 }
 
+/**
+ * Registers statistical data for a sent packet in a given stateless conversation (two IPs, two ports). 
+ * Increments the counter packets_A_B or packets_B_A.
+ * Adds the timestamp of the packet in pkts_A_B_timestamp or pkts_B_A_timestamp.
+ * @param ipAddressSender The sender IP address.
+ * @param sport The source port.
+ * @param ipAddressReceiver The receiver IP address.
+ * @param dport The destination port.
+ * @param timestamp The timestamp of the packet.
+ */
+void statistics::addConvStatStateless(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp){
+
+    conv f1 = {ipAddressReceiver, dport, ipAddressSender, sport};
+    conv f2 = {ipAddressSender, sport, ipAddressReceiver, dport};
+
+    // if already exist A(ipAddressReceiver, dport), B(ipAddressSender, sport) conversation
+    if (conv_statistics_stateless.count(f1)>0){
+        conv_statistics_stateless[f1].pkts_count++;
+        if(conv_statistics_stateless[f1].pkts_count<=3)
+            conv_statistics_stateless[f1].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics_stateless[f1].pkts_timestamp.back()));
+        conv_statistics_stateless[f1].pkts_timestamp.push_back(timestamp);
+    }
+    // Add new conversation A(ipAddressSender, sport), B(ipAddressReceiver, dport)
+    else{
+        conv_statistics_stateless[f2].pkts_count++;
+        if(conv_statistics_stateless[f2].pkts_timestamp.size()>0 && conv_statistics_stateless[f2].pkts_count<=3 )
+            conv_statistics_stateless[f2].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics_stateless[f2].pkts_timestamp.back()));
+        conv_statistics_stateless[f2].pkts_timestamp.push_back(timestamp);
+    }
+}
+
 /**
  * Increments the packet counter for the given IP address and MSS value.
  * @param ipAddress The IP address whose MSS packet counter should be incremented.
@@ -601,6 +632,7 @@ void statistics::writeToDatabase(std::string database_path) {
         db.writeStatisticsToS(tos_distribution);
         db.writeStatisticsWin(win_distribution);
         db.writeStatisticsConv(conv_statistics);
+        db.writeStatisticsConvStateless(conv_statistics_stateless);
         db.writeStatisticsInterval(interval_statistics);
     }
     else {

code_boost/src/cxx/statistics.h

@@ -373,6 +373,8 @@ public:
 
     void addConvStat(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
 
+    void addConvStatStateless(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
+
     std::vector<float> calculateIPsCumEntropy();
 
     std::vector<float> calculateLastIntervalIPsEntropy(std::chrono::microseconds intervalStartTimestamp);
@@ -487,6 +489,11 @@ private:
     // average of inter-arrival times}
     std::unordered_map<conv, entry_convStat> conv_statistics;
 
+    // {IP Address A, Port A, IP Address B, Port B,   #packets, packets timestamps, inter-arrival times,
+    // average of inter-arrival times}
+    // Also stores conversation with only one exchanged message. In this case avgPktRate, minDelay, maxDelay and avgDelay are -1
+    std::unordered_map<conv, entry_convStat> conv_statistics_stateless;
+
     // {Last timestamp in the interval, #packets, #bytes, source IP entropy, destination IP entropy,
     // source IP cumulative entropy, destination IP cumulative entropy, #payload, #incorrect TCP checksum,
     // #correct TCP checksum, #novel IP, #novel TTL, #novel Window Size, #novel ToS,#novel MSS}

code_boost/src/cxx/statistics_db.cpp

@@ -372,24 +372,90 @@ void statistics_db::writeStatisticsConv(std::unordered_map<conv, entry_convStat>
                 query.exec();
                 query.reset();
             }
-            else if (e.pkts_count == 1){
-                int minDelay = -1;
-                int maxDelay = -1;
-                e.avg_pkt_rate = (float) -1;
-                e.avg_interarrival_time = e.avg_interarrival_time = (std::chrono::microseconds) 0;
+        }
+        transaction.commit();
+    }
+    catch (std::exception &e) {
+        std::cout << "Exception in statistics_db: " << e.what() << std::endl;
+    }
+}
 
+
+/**
+ * Writes the stateless conversation statistics into the database.
+ * @param convStatistics The stateless conversation from class statistics.
+ */
+void statistics_db::writeStatisticsConvStateless(std::unordered_map<conv, entry_convStat> convStatistics){          
+    try {
+        db->exec("DROP TABLE IF EXISTS conv_statistics_stateless");
+        SQLite::Transaction transaction(*db);
+        const char *createTable = "CREATE TABLE conv_statistics_stateless ("
+                "ipAddressA TEXT,"
+                "portA INTEGER,"
+                "ipAddressB TEXT,"              
+                "portB INTEGER,"
+                "pktsCount INTEGER,"
+                "avgPktRate REAL,"
+                "avgDelay INTEGER,"
+                "minDelay INTEGER,"
+                "maxDelay INTEGER,"
+                "PRIMARY KEY(ipAddressA,portA,ipAddressB,portB));";
+        db->exec(createTable);
+        SQLite::Statement query(*db, "INSERT INTO conv_statistics_stateless VALUES (?, ?, ?, ?, ?,  ?, ?, ?, ?)");
+
+        // Calculate average of inter-arrival times and average packet rate
+        for (auto it = convStatistics.begin(); it != convStatistics.end(); ++it) {
+            conv f = it->first;
+            entry_convStat e = it->second;
+            if (e.pkts_count > 0){
                 query.bind(1, f.ipAddressA);
                 query.bind(2, f.portA);
                 query.bind(3, f.ipAddressB);
                 query.bind(4, f.portB);
-                query.bind(5, (int) e.pkts_count);
-                query.bind(6, (float) e.avg_pkt_rate);
-                query.bind(7, (int) e.avg_interarrival_time.count());
-                query.bind(8, minDelay);
-                query.bind(9, maxDelay);
-                query.exec();
-                query.reset();
+
+                if (e.pkts_count == 1){
+                    e.avg_pkt_rate = (float) -1;
+                    e.avg_interarrival_time = (std::chrono::microseconds) -1;
+
+                    query.bind(5, (int) e.pkts_count);
+                    query.bind(6, (float) e.avg_pkt_rate);
+                    query.bind(7, (int) e.avg_interarrival_time.count());
+                    query.bind(8, -1);
+                    query.bind(9, -1);
+                    query.exec();
+                    query.reset();
+                }
+                else {
+                    int sumDelay = 0;
+                    int minDelay = -1;
+                    int maxDelay = -1;
+                    for (int i = 0; (unsigned) i < e.interarrival_time.size(); i++) {
+                        sumDelay += e.interarrival_time[i].count();
+                        if (maxDelay < e.interarrival_time[i].count())
+                            maxDelay = e.interarrival_time[i].count();
+                        if (minDelay > e.interarrival_time[i].count() || minDelay == -1)
+                            minDelay = e.interarrival_time[i].count();
+                    }
+                    if (e.interarrival_time.size() > 0)
+                        e.avg_interarrival_time = (std::chrono::microseconds) sumDelay / e.interarrival_time.size(); // average
+                    else e.avg_interarrival_time = (std::chrono::microseconds) 0;
+
+                    std::chrono::microseconds start_timesttamp = e.pkts_timestamp[0];
+                    std::chrono::microseconds end_timesttamp = e.pkts_timestamp.back();
+                    std::chrono::microseconds conn_duration = end_timesttamp - start_timesttamp;
+                    e.avg_pkt_rate = (float) e.pkts_count * 1000000 / conn_duration.count(); // pkt per sec
+
+                    
+                    query.bind(5, (int) e.pkts_count);
+                    query.bind(6, (float) e.avg_pkt_rate);
+                    query.bind(7, (int) e.avg_interarrival_time.count());
+                    query.bind(8, minDelay);
+                    query.bind(9, maxDelay);
+                    query.exec();
+                    query.reset();
+                }
             }
+            
         }
         transaction.commit();
     }
@@ -398,6 +464,7 @@ void statistics_db::writeStatisticsConv(std::unordered_map<conv, entry_convStat>
     }
 }
 
+
 /**
  * Writes the interval statistics into the database.
  * @param intervalStatistics The interval entries from class statistics.

code_boost/src/cxx/statistics_db.h

@@ -43,6 +43,8 @@ public:
 
     void writeStatisticsConv(std::unordered_map<conv, entry_convStat> convStatistics);
 
+    void writeStatisticsConvStateless(std::unordered_map<conv, entry_convStat> convStatistics);
+
     void writeStatisticsInterval(std::unordered_map<std::string, entry_intervalStat> intervalStatistics);
 
 private: