count the new values of IP,TTL,ToS,MSS,Win Size, in every interval, write to db, and plot dist

code/ID2TLib/Statistics.py

@@ -635,6 +635,171 @@ class Statistics:
             plt.savefig(out, dpi=500)
             return out
 
+        # Aidmar
+        def plot_interval_new_ip(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("IP New Values Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('New values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-new-ip-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        # Aidmar
+        def plot_interval_new_ttl(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("TTL New Values Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('New values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-new-ttl-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        # Aidmar
+        def plot_interval_new_tos(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("ToS New Values Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('New values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-new-tos-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        # Aidmar
+        def plot_interval_new_win_size(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Window Size New Values Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('New values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-new-win-size-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        # Aidmar
+        def plot_interval_new_mss(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+
+            plt.autoscale(enable=True, axis='both')
+            plt.title("MSS New Values Distribution")
+            plt.xlabel('Timestamp')
+            plt.ylabel('New values count')
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+            width = 0.5
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=2, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-interval-new-mss-dist' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
         ttl_out_path = plot_ttl('.' + format)
         mss_out_path = plot_mss('.' + format)
         win_out_path = plot_win('.' + format)
@@ -647,7 +812,11 @@ class Statistics:
         plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + format)
         plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + format)
         plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + format)
-
+        plot_interval_new_ip = plot_interval_new_ip('.' + format)
+        plot_interval_new_ttl = plot_interval_new_ttl('.' + format)
+        plot_interval_new_tos = plot_interval_new_tos('.' + format)
+        plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
+        plot_interval_new_mss = plot_interval_new_mss('.' + format)
 
         #print("Saved distributions plots at: %s, %s, %s, %s, %s, %s, %s, %s %s" %(ttl_out_path,mss_out_path, win_out_path,
         #protocol_out_path, port_out_path,ip_src_out_path,ip_dst_out_path, plot_interval_pktCount))

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/CXX.includecache

@@ -58,28 +58,6 @@ map
 SQLiteCpp/Exception.h
 -
 
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp
-iostream
--
-fstream
--
-sstream
--
-artifacts_tests.h
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.h
-utilities.h
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/utilities.h
-
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.h
-tins/tins.h
--
-iostream
--
-stdio.h
--
-utilities.h
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/utilities.h
-
 /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/pcap_processor.cpp
 pcap_processor.h
 /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/pcap_processor.h
@@ -101,8 +79,6 @@ unordered_map
 -
 statistics.h
 /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/statistics.h
-artifacts_tests.h
-/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.h
 
 /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/statistics.cpp
 iostream
@@ -179,4 +155,8 @@ algorithm
 -
 tins/tins.h
 -
+bitset
+-
+type_traits
+-
 

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/DependInfo.cmake

@@ -4,7 +4,6 @@ set(CMAKE_DEPENDS_LANGUAGES
   )
 # The set of files for implicit dependencies of each language:
 set(CMAKE_DEPENDS_CHECK_CXX
-  "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp" "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o"
   "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/pcap_processor.cpp" "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o"
   "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/statistics.cpp" "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o"
   "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/statistics_db.cpp" "/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o"

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/build.make

@@ -129,33 +129,9 @@ CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o.provides: CMakeFiles/cpp-p
 CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o.provides.build: CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o
 
 
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o: CMakeFiles/cpp-pcapreader.dir/flags.make
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o: ../cxx/artifacts_tests.cpp
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --progress-dir=/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles --progress-num=$(CMAKE_PROGRESS_4) "Building CXX object CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o"
-	/usr/bin/c++  $(CXX_DEFINES) $(CXX_INCLUDES) $(CXX_FLAGS) -o CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o -c /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp
-
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.i: cmake_force
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green "Preprocessing CXX source to CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.i"
-	/usr/bin/c++ $(CXX_DEFINES) $(CXX_INCLUDES) $(CXX_FLAGS) -E /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp > CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.i
-
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.s: cmake_force
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green "Compiling CXX source to assembly CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.s"
-	/usr/bin/c++ $(CXX_DEFINES) $(CXX_INCLUDES) $(CXX_FLAGS) -S /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp -o CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.s
-
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.requires:
-
-.PHONY : CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.requires
-
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.provides: CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.requires
-	$(MAKE) -f CMakeFiles/cpp-pcapreader.dir/build.make CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.provides.build
-.PHONY : CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.provides
-
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.provides.build: CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o
-
-
 CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o: CMakeFiles/cpp-pcapreader.dir/flags.make
 CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o: ../cxx/utilities.cpp
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --progress-dir=/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles --progress-num=$(CMAKE_PROGRESS_5) "Building CXX object CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o"
+	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --progress-dir=/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles --progress-num=$(CMAKE_PROGRESS_4) "Building CXX object CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o"
 	/usr/bin/c++  $(CXX_DEFINES) $(CXX_INCLUDES) $(CXX_FLAGS) -o CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o -c /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/utilities.cpp
 
 CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.i: cmake_force
@@ -182,7 +158,6 @@ cpp__pcapreader_OBJECTS = \
 "CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o" \
 "CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o" \
 "CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o" \
-"CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o" \
 "CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o"
 
 # External object files for target cpp-pcapreader
@@ -191,7 +166,6 @@ cpp__pcapreader_EXTERNAL_OBJECTS =
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o
-cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/build.make
 cpp-pcapreader: /usr/lib/x86_64-linux-gnu/libboost_python-py35.so
@@ -199,7 +173,7 @@ cpp-pcapreader: /usr/local/lib/libtins.so
 cpp-pcapreader: SQLiteCpp/libSQLiteCpp.a
 cpp-pcapreader: SQLiteCpp/sqlite3/libsqlite3.a
 cpp-pcapreader: CMakeFiles/cpp-pcapreader.dir/link.txt
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --bold --progress-dir=/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles --progress-num=$(CMAKE_PROGRESS_6) "Linking CXX executable cpp-pcapreader"
+	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --green --bold --progress-dir=/home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cmake-build-debug/CMakeFiles --progress-num=$(CMAKE_PROGRESS_5) "Linking CXX executable cpp-pcapreader"
 	$(CMAKE_COMMAND) -E cmake_link_script CMakeFiles/cpp-pcapreader.dir/link.txt --verbose=$(VERBOSE)
 
 # Rule to build all files generated by this target.
@@ -210,7 +184,6 @@ CMakeFiles/cpp-pcapreader.dir/build: cpp-pcapreader
 CMakeFiles/cpp-pcapreader.dir/requires: CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o.requires
 CMakeFiles/cpp-pcapreader.dir/requires: CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o.requires
 CMakeFiles/cpp-pcapreader.dir/requires: CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o.requires
-CMakeFiles/cpp-pcapreader.dir/requires: CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o.requires
 CMakeFiles/cpp-pcapreader.dir/requires: CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o.requires
 
 .PHONY : CMakeFiles/cpp-pcapreader.dir/requires

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/cmake_clean.cmake

@@ -2,7 +2,6 @@ file(REMOVE_RECURSE
   "CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o"
   "CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o"
   "CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o"
-  "CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o"
   "CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o"
   "cpp-pcapreader.pdb"
   "cpp-pcapreader"

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/depend.internal

@@ -1,12 +1,7 @@
 # CMAKE generated file: DO NOT EDIT!
 # Generated by "Unix Makefiles" Generator, CMake Version 3.8
 
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o
- /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.cpp
- /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.h
- /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/utilities.h
 CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o
- /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/artifacts_tests.h
  /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/pcap_processor.cpp
  /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/pcap_processor.h
  /home/anonymous/Downloads/ID2T-toolkit/code_boost/src/cxx/statistics.h

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/depend.make

@@ -1,11 +1,6 @@
 # CMAKE generated file: DO NOT EDIT!
 # Generated by "Unix Makefiles" Generator, CMake Version 3.8
 
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o: ../cxx/artifacts_tests.cpp
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o: ../cxx/artifacts_tests.h
-CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o: ../cxx/utilities.h
-
-CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o: ../cxx/artifacts_tests.h
 CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o: ../cxx/pcap_processor.cpp
 CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o: ../cxx/pcap_processor.h
 CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o: ../cxx/statistics.h

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/link.txt

@@ -1 +1 @@
-/usr/bin/c++  -g -Wall  -rdynamic CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/artifacts_tests.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o  -o cpp-pcapreader -Wl,-rpath,/usr/local/lib -lboost_python-py35 /usr/local/lib/libtins.so SQLiteCpp/libSQLiteCpp.a SQLiteCpp/sqlite3/libsqlite3.a -lpthread -ldl 
+/usr/bin/c++  -g -Wall  -rdynamic CMakeFiles/cpp-pcapreader.dir/cxx/pcap_processor.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/statistics.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/statistics_db.cpp.o CMakeFiles/cpp-pcapreader.dir/cxx/utilities.cpp.o  -o cpp-pcapreader -Wl,-rpath,/usr/local/lib -lboost_python-py35 /usr/local/lib/libtins.so SQLiteCpp/libSQLiteCpp.a SQLiteCpp/sqlite3/libsqlite3.a -lpthread -ldl 

code_boost/src/cmake-build-debug/CMakeFiles/cpp-pcapreader.dir/progress.make

@@ -3,5 +3,4 @@ CMAKE_PROGRESS_2 = 9
 CMAKE_PROGRESS_3 = 10
 CMAKE_PROGRESS_4 = 11
 CMAKE_PROGRESS_5 = 12
-CMAKE_PROGRESS_6 = 13
 

code_boost/src/cxx/pcap_processor.cpp

@@ -151,12 +151,14 @@ void pcap_processor::collect_statistics() {
 
             // For each interval
             if(currentCaptureDuration>barrier && barrier.count() > 0){ // barrier becomes negative in last interval
-                stats.addIntervalStat(timeInterval, intervalStartTimestamp, lastPktTimestamp, previousPacketCount, previousSumPacketSize);
+                stats.addIntervalStat(timeInterval, intervalStartTimestamp, lastPktTimestamp);
                 timeIntervalCounter++;
                 barrier =  barrier+timeInterval;
                 intervalStartTimestamp = lastPktTimestamp;
+
                 previousPacketCount = stats.getPacketCount();
                 previousSumPacketSize = stats.getSumPacketSize();
+
             }
             stats.incrementPacketCount();
             this->process_packets(*i);                    
@@ -167,9 +169,9 @@ void pcap_processor::collect_statistics() {
         stats.setTimestampLastPacket(lastProcessedPacket);
 
         // TO-DO: to delete
-        for (auto it = stats.dscp_distribution.begin(); it != stats.dscp_distribution.end(); ++it) {
-            std::cout<<it->first<<","<<it->second<<"\n";
-        }
+        //for (auto it = stats.dscp_distribution.begin(); it != stats.dscp_distribution.end(); ++it) {
+        //    std::cout<<it->first<<","<<it->second<<"\n";
+        //}
     }
 }
 
@@ -208,7 +210,10 @@ void pcap_processor::process_packets(const Packet &pkt) {
         stats.addIpStat_packetSent(filePath, ipAddressSender, ipLayer.dst_addr().to_string(), sizeCurrentPacket, pkt.timestamp());
 
         // TTL distribution
-        stats.incrementTTLcount(ipAddressSender, ipLayer.ttl());      
+        stats.incrementTTLcount(ipAddressSender, ipLayer.ttl());
+
+        // Aidmar - ToS distribution
+        stats.incrementToScount(ipAddressSender, ipLayer.tos());
 
         // Protocol distribution
         stats.incrementProtocolCount(ipAddressSender, "IPv4");
@@ -326,7 +331,7 @@ bool inline pcap_processor::file_exists(const std::string &filePath) {
  */
 //int main() {
 //    std::cout << "Starting application." << std::endl;
-//    pcap_processor pcap = pcap_processor("/home/anonymous/Downloads/ID2T-toolkit/captures/iscx/1h_iscx_11jun.pcap", "False");
+//    pcap_processor pcap = pcap_processor("/home/anonymous/Downloads/ID2T-toolkit/captures/col/capture_1.pcap", "True");
 //
 //    long double t = pcap.get_timestamp_mu_sec(87);
 //    std::cout << t << std::endl;

code_boost/src/cxx/statistics.cpp

@@ -208,32 +208,44 @@ void statistics::calculateIPIntervalPacketRate(std::chrono::duration<int, std::m
  * @param intervalEndTimestamp The timstamp where the interval ends.
  * @param previousPacketCount The total number of packets in last interval.
  */
-void statistics::addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds intervalEndTimestamp, int previousPacketCount, float previousSumPacketSize){
+void statistics::addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds intervalEndTimestamp){
     // Add packet rate for each IP to ip_statistics map
     calculateIPIntervalPacketRate(interval, intervalStartTimestamp);
     
     std::vector<float> ipEntopies = calculateLastIntervalIPsEntropy(intervalStartTimestamp);
     std::vector<float> ipCumEntopies = calculateIPsCumEntropy();
     std::string lastPktTimestamp_s = std::to_string(intervalEndTimestamp.count());
-
-    interval_statistics[lastPktTimestamp_s].pkts_count = packetCount - previousPacketCount;  
-    interval_statistics[lastPktTimestamp_s].kbytes = (float(sumPacketSize - previousSumPacketSize) / 1024);
-
-    interval_statistics[lastPktTimestamp_s].payload_count = payloadCount;
-    interval_statistics[lastPktTimestamp_s].incorrect_checksum_count = incorrectTCPChecksumCount;
-    interval_statistics[lastPktTimestamp_s].correct_checksum_count = correctTCPChecksumCount;
-    interval_statistics[lastPktTimestamp_s].invalid_tos_count = invalidToSCount;
-    interval_statistics[lastPktTimestamp_s].valid_tos_count = validToSCount;
-
-    std::cout<<invalidToSCount<<","<<validToSCount<<"\n";
-
-
-    // Reset variables for next interval
-    payloadCount = 0;
-    incorrectTCPChecksumCount = 0;
-    correctTCPChecksumCount = 0;
-    invalidToSCount = 0;
-    validToSCount = 0;
+    std::string  intervalStartTimestamp_s = std::to_string(intervalStartTimestamp.count());
+
+    // The intervalStartTimestamp_s is the previous interval lastPktTimestamp_s
+    interval_statistics[lastPktTimestamp_s].pkts_count = packetCount - lastIntervalCumPktCount;
+    interval_statistics[lastPktTimestamp_s].kbytes = (float(sumPacketSize - lastIntervalCumSumPktSize) / 1024);
+
+    interval_statistics[lastPktTimestamp_s].payload_count = payloadCount - lastIntervalPayloadCount;
+    interval_statistics[lastPktTimestamp_s].incorrect_checksum_count = incorrectTCPChecksumCount - lastIntervalIncorrectTCPChecksumCount;
+    interval_statistics[lastPktTimestamp_s].correct_checksum_count = correctTCPChecksumCount - lastIntervalCorrectTCPChecksumCount;
+    interval_statistics[lastPktTimestamp_s].invalid_tos_count = invalidToSCount - lastIntervalInvalidToSCount;
+    interval_statistics[lastPktTimestamp_s].valid_tos_count = validToSCount - lastIntervalValidToSCount;
+    interval_statistics[lastPktTimestamp_s].new_ip_count = ip_statistics.size() - lastIntervalCumNewIPCount;
+    interval_statistics[lastPktTimestamp_s].new_ttl_count = ttl_distribution.size() - lastIntervalCumNewTTLCount;
+    interval_statistics[lastPktTimestamp_s].new_win_size_count = win_distribution.size() - lastIntervalCumNewWinSizeCount;
+    interval_statistics[lastPktTimestamp_s].new_tos_count = tos_distribution.size() - lastIntervalCumNewToSCount;
+    interval_statistics[lastPktTimestamp_s].new_mss_count = mss_distribution.size() - lastIntervalCumNewMSSCount;
+
+    //std::cout<<invalidToSCount<<","<<validToSCount<<"\n";
+
+    lastIntervalPayloadCount = payloadCount;
+    lastIntervalIncorrectTCPChecksumCount = incorrectTCPChecksumCount;
+    lastIntervalCorrectTCPChecksumCount = correctTCPChecksumCount;
+    lastIntervalInvalidToSCount = invalidToSCount;
+    lastIntervalValidToSCount = validToSCount;
+    lastIntervalCumPktCount = packetCount;
+    lastIntervalCumSumPktSize = sumPacketSize;
+    lastIntervalCumNewIPCount =  ip_statistics.size();
+    lastIntervalCumNewTTLCount = ttl_distribution.size();
+    lastIntervalCumNewWinSizeCount = win_distribution.size();
+    lastIntervalCumNewToSCount = tos_distribution.size();
+    lastIntervalCumNewMSSCount = mss_distribution.size();
 
     if(ipEntopies.size()>1){
         interval_statistics[lastPktTimestamp_s].ip_src_entropy = ipEntopies[0];
@@ -307,6 +319,15 @@ void statistics::incrementTTLcount(std::string ipAddress, int ttlValue) {
     ttl_distribution[{ipAddress, ttlValue}]++;
 }
 
+/**
+ * Increments the packet counter for the given IP address and ToS value.
+ * @param ipAddress The IP address whose ToS packet counter should be incremented.
+ * @param tosValue The ToS value of the packet.
+ */
+void statistics::incrementToScount(std::string ipAddress, int tosValue) {
+    tos_distribution[{ipAddress, tosValue}]++;
+}
+
 /**
  * Increments the protocol counter for the given IP address and protocol.
  * @param ipAddress The IP address whose protocol packet counter should be incremented.

code_boost/src/cxx/statistics.h

@@ -82,6 +82,22 @@ struct ipAddress_mss {
     }
 };
 
+// Aidmar
+/*
+ * Struct used to represent:
+ * - IP address (IPv4 or IPv6)
+ * - ToS value
+ */
+struct ipAddress_tos {
+    std::string ipAddress;
+    int tosValue;
+
+    bool operator==(const ipAddress_tos &other) const {
+        return ipAddress == other.ipAddress
+               && tosValue == other.tosValue;
+    }
+};
+
 // Aidmar
 /*
  * Struct used to represent:
@@ -196,6 +212,12 @@ struct entry_intervalStat {
     int correct_checksum_count;
     int invalid_tos_count;
     int valid_tos_count;
+    int new_ip_count;
+    int new_ttl_count;
+    int new_win_size_count;
+    int new_tos_count;
+    int new_mss_count;
+
     // Predictability score
     //float ip_src_pred_score;
     //float ip_dst_pred_score;
@@ -211,7 +233,12 @@ struct entry_intervalStat {
                && payload_count == other.payload_count
                && incorrect_checksum_count == other.incorrect_checksum_count
                && invalid_tos_count == other.invalid_tos_count
-               && valid_tos_count == other.valid_tos_count;
+               && valid_tos_count == other.valid_tos_count
+               && new_ip_count == other.new_ip_count
+               && new_ttl_count == other.new_ttl_count
+               && new_win_size_count == other.new_win_size_count
+               && new_tos_count == other.new_tos_count
+               && new_mss_count == other.new_mss_count;
     }
 };
 
@@ -285,6 +312,18 @@ namespace std {
         }
     };
 
+    // Aidmar
+    template<>
+    struct hash<ipAddress_tos> {
+        std::size_t operator()(const ipAddress_tos &k) const {
+            using std::size_t;
+            using std::hash;
+            using std::string;
+            return ((hash<string>()(k.ipAddress)
+                     ^ (hash<int>()(k.tosValue) << 1)) >> 1);
+        }
+    };
+
     // Aidmar
       template<>
     struct hash<ipAddress_win> {
@@ -359,10 +398,11 @@ public:
     void addConvStat(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
     std::vector<float> calculateIPsCumEntropy();
     std::vector<float> calculateLastIntervalIPsEntropy(std::chrono::microseconds intervalStartTimestamp);        
-    void addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds lastPktTimestamp, int previousPacketCount, float previousSumPacketSize);
+    void addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds lastPktTimestamp);
     void checkPayload(const PDU *pdu_l4);
     void checkTCPChecksum(std::string ipAddressSender, std::string ipAddressReceiver, TCP tcpPkt);
     void checkToS(uint8_t ToS);
+    void incrementToScount(std::string ipAddress, int tosValue);
 
     void incrementTTLcount(std::string ipAddress, int ttlValue);
 
@@ -424,12 +464,28 @@ private:
 
     // Aidmar
     bool doTests = false;
+
     int payloadCount = 0;
     int incorrectTCPChecksumCount = 0;
     int correctTCPChecksumCount = 0;
     int validToSCount = 0;
     int invalidToSCount = 0;
 
+    int lastIntervalPayloadCount = 0;
+    int lastIntervalIncorrectTCPChecksumCount = 0;
+    int lastIntervalCorrectTCPChecksumCount = 0;
+    int lastIntervalValidToSCount = 0;
+    int lastIntervalInvalidToSCount = 0;
+    int lastIntervalCumPktCount = 0;
+    float lastIntervalCumSumPktSize = 0;
+    int lastIntervalCumNewIPCount = 0;
+    int lastIntervalCumNewTTLCount = 0;
+    int lastIntervalCumNewWinSizeCount = 0;
+    int lastIntervalCumNewToSCount = 0;
+    int lastIntervalCumNewMSSCount = 0;
+
+
+
     /*
      * Data containers
      */
@@ -444,7 +500,7 @@ private:
     // {IP Address A, Port A, IP Address B, Port B,   #packets_A_B, #packets_B_A}
     std::unordered_map<conv, entry_convStat> conv_statistics;
     std::unordered_map<std::string, entry_intervalStat> interval_statistics;
-
+    std::unordered_map<ipAddress_tos, int> tos_distribution;
 
     // {IP Address, Protocol, count}
     std::unordered_map<ipAddress_protocol, int> protocol_distribution;

code_boost/src/cxx/statistics_db.cpp

@@ -394,9 +394,14 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
                 "payloadCount INTEGER,"
                 "incorrectTCPChecksumCount INTEGER,"
                 "correctTCPChecksumCount INTEGER,"
+                "newIPCount INTEGER,"
+                "newTTLCount INTEGER,"
+                "newWinSizeCount INTEGER,"
+                "newToSCount INTEGER,"
+                "newMSSCount INTEGER,"
                 "PRIMARY KEY(lastPktTimestamp));";
         db->exec(createTable);
-        SQLite::Statement query(*db, "INSERT INTO interval_statistics VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+        SQLite::Statement query(*db, "INSERT INTO interval_statistics VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
         for (auto it = intervalStatistics.begin(); it != intervalStatistics.end(); ++it) {
             std::string t = it->first;
             entry_intervalStat e = it->second;        
@@ -411,6 +416,11 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
             query.bind(8, e.payload_count);
             query.bind(9, e.incorrect_checksum_count);
             query.bind(10, e.correct_checksum_count);
+            query.bind(11, e.new_ip_count);
+            query.bind(12, e.new_ttl_count);
+            query.bind(13, e.new_win_size_count);
+            query.bind(14, e.new_tos_count);
+            query.bind(15, e.new_mss_count);
             query.exec();
             query.reset();
         }