add port novelty distribution

code/ID2TLib/Statistics.py

@@ -954,6 +954,37 @@ class Statistics:
             plt.savefig(out, dpi=500)
             return out
 
+        def plot_interval_new_port(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newPortCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Port Novelty Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('Novel values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-novel-port-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
         def plot_interval_new_ttl(file_ending: str):
             plt.gcf().clear()
             result = self.stats_db._process_user_defined_query(
@@ -1100,6 +1131,7 @@ class Statistics:
         plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + format)
         plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + format)
         plot_interval_new_ip = plot_interval_new_ip('.' + format)
+        plot_interval_new_port = plot_interval_new_port('.' + format)
         plot_interval_new_ttl = plot_interval_new_ttl('.' + format)
         plot_interval_new_tos = plot_interval_new_tos('.' + format)
         plot_interval_new_win_size = plot_interval_new_win_size('.' + format)

code_boost/src/cxx/statistics.cpp

@@ -186,22 +186,25 @@ void statistics::addIntervalStat(std::chrono::duration<int, std::micro> interval
     interval_statistics[lastPktTimestamp_s].payload_count = payloadCount - intervalPayloadCount;
     interval_statistics[lastPktTimestamp_s].incorrect_tcp_checksum_count = incorrectTCPChecksumCount - intervalIncorrectTCPChecksumCount;
     interval_statistics[lastPktTimestamp_s].correct_tcp_checksum_count = correctTCPChecksumCount - intervalCorrectTCPChecksumCount;
-    interval_statistics[lastPktTimestamp_s].novel_ip_count = ip_statistics.size() - intervalCumNewIPCount;
-    interval_statistics[lastPktTimestamp_s].novel_ttl_count = ttl_values.size() - intervalCumNewTTLCount;
-    interval_statistics[lastPktTimestamp_s].novel_win_size_count = win_values.size() - intervalCumNewWinSizeCount;
-    interval_statistics[lastPktTimestamp_s].novel_tos_count = tos_values.size() - intervalCumNewToSCount;
-    interval_statistics[lastPktTimestamp_s].novel_mss_count = mss_values.size() - intervalCumNewMSSCount;
+    interval_statistics[lastPktTimestamp_s].novel_ip_count = ip_statistics.size() - intervalCumNovelIPCount;
+    interval_statistics[lastPktTimestamp_s].novel_ttl_count = ttl_values.size() - intervalCumNovelTTLCount;
+    interval_statistics[lastPktTimestamp_s].novel_win_size_count = win_values.size() - intervalCumNovelWinSizeCount;
+    interval_statistics[lastPktTimestamp_s].novel_tos_count = tos_values.size() - intervalCumNovelToSCount;
+    interval_statistics[lastPktTimestamp_s].novel_mss_count = mss_values.size() - intervalCumNovelMSSCount;
+    interval_statistics[lastPktTimestamp_s].novel_port_count = port_values.size() - intervalCumNovelPortCount;
+
 
     intervalPayloadCount = payloadCount;
     intervalIncorrectTCPChecksumCount = incorrectTCPChecksumCount;
     intervalCorrectTCPChecksumCount = correctTCPChecksumCount;
     intervalCumPktCount = packetCount;
     intervalCumSumPktSize = sumPacketSize;
-    intervalCumNewIPCount =  ip_statistics.size();
-    intervalCumNewTTLCount = ttl_values.size();
-    intervalCumNewWinSizeCount = win_values.size();
-    intervalCumNewToSCount = tos_values.size();
-    intervalCumNewMSSCount = mss_values.size();
+    intervalCumNovelIPCount =  ip_statistics.size();
+    intervalCumNovelTTLCount = ttl_values.size();
+    intervalCumNovelWinSizeCount = win_values.size();
+    intervalCumNovelToSCount = tos_values.size();
+    intervalCumNovelMSSCount = mss_values.size();
+    intervalCumNovelPortCount = port_values.size();
 
     if(ipEntopies.size()>1){
         interval_statistics[lastPktTimestamp_s].ip_src_entropy = ipEntopies[0];
@@ -314,8 +317,11 @@ int statistics::getProtocolCount(std::string ipAddress, std::string protocol) {
  */
 void statistics::incrementPortCount(std::string ipAddressSender, int outgoingPort, std::string ipAddressReceiver,
                                     int incomingPort) {
+    port_values[outgoingPort]++;
+    port_values[incomingPort]++;
     ip_ports[{ipAddressSender, "out", outgoingPort}]++;
     ip_ports[{ipAddressReceiver, "in", incomingPort}]++;
+
 }
 
 /**

code_boost/src/cxx/statistics.h

@@ -201,6 +201,7 @@ struct entry_intervalStat {
     int novel_win_size_count;
     int novel_tos_count;
     int novel_mss_count;
+    int novel_port_count;
 
     bool operator==(const entry_intervalStat &other) const {
         return pkts_count == other.pkts_count
@@ -215,7 +216,8 @@ struct entry_intervalStat {
                && novel_ttl_count == other.novel_ttl_count
                && novel_win_size_count == other.novel_win_size_count
                && novel_tos_count == other.novel_tos_count
-               && novel_mss_count == other.novel_mss_count;
+               && novel_mss_count == other.novel_mss_count
+               && novel_port_count == other.novel_port_count;
     }
 };
 
@@ -459,11 +461,12 @@ private:
     int intervalCorrectTCPChecksumCount = 0;
     int intervalCumPktCount = 0;
     float intervalCumSumPktSize = 0;
-    int intervalCumNewIPCount = 0;
-    int intervalCumNewTTLCount = 0;
-    int intervalCumNewWinSizeCount = 0;
-    int intervalCumNewToSCount = 0;
-    int intervalCumNewMSSCount = 0;
+    int intervalCumNovelIPCount = 0;
+    int intervalCumNovelTTLCount = 0;
+    int intervalCumNovelWinSizeCount = 0;
+    int intervalCumNovelToSCount = 0;
+    int intervalCumNovelMSSCount = 0;
+    int intervalCumNovelPortCount = 0;
 
     /*
      * Data containers
@@ -501,6 +504,9 @@ private:
     // {MSS, count}
     std::unordered_map<int, int> mss_values;
 
+    // {Port, count}
+    std::unordered_map<int, int> port_values;
+
     // {IP Address, Protocol, count}
     std::unordered_map<ipAddress_protocol, int> protocol_distribution;
 

code_boost/src/cxx/statistics_db.cpp

@@ -400,13 +400,14 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
                 "incorrectTCPChecksumCount INTEGER,"
                 "correctTCPChecksumCount INTEGER,"
                 "newIPCount INTEGER,"
+                "newPortCount INTEGER,"
                 "newTTLCount INTEGER,"
                 "newWinSizeCount INTEGER,"
                 "newToSCount INTEGER,"
                 "newMSSCount INTEGER,"
                 "PRIMARY KEY(lastPktTimestamp));";
         db->exec(createTable);
-        SQLite::Statement query(*db, "INSERT INTO interval_statistics VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+        SQLite::Statement query(*db, "INSERT INTO interval_statistics VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
         for (auto it = intervalStatistics.begin(); it != intervalStatistics.end(); ++it) {
             std::string t = it->first;
             entry_intervalStat e = it->second;        
@@ -422,10 +423,11 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
             query.bind(9, e.incorrect_tcp_checksum_count);
             query.bind(10, e.correct_tcp_checksum_count);
             query.bind(11, e.novel_ip_count);
-            query.bind(12, e.novel_ttl_count);
-            query.bind(13, e.novel_win_size_count);
-            query.bind(14, e.novel_tos_count);
-            query.bind(15, e.novel_mss_count);
+            query.bind(12, e.novel_port_count);
+            query.bind(13, e.novel_ttl_count);
+            query.bind(14, e.novel_win_size_count);
+            query.bind(15, e.novel_tos_count);
+            query.bind(16, e.novel_mss_count);
             query.exec();
             query.reset();
         }