Rewrote first PCAP parsing pass for performance

Reduces time needed for MAWI dataset by ~40 seconds

code_boost/src/CMakeLists.txt

@@ -81,7 +81,7 @@ SET_target_properties(sqlite3 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 ADD_LIBRARY(pcapreader SHARED ${SOURCE_FILES})
 # Libs pthread and dl are prerequisites of SQLiteCpp
-TARGET_LINK_LIBRARIES(pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl)
+TARGET_LINK_LIBRARIES(pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl pcap)
 
 ADD_LIBRARY(botnetcomm SHARED ${BOT_COMM_PROC_SOURCE})
 TARGET_LINK_LIBRARIES(botnetcomm ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})

code_boost/src/cxx/pcap_processor.cpp

@@ -112,6 +112,44 @@ std::string pcap_processor::merge_pcaps(const std::string pcap_path) {
     return new_filepath;
 }
 
+bool pcap_processor::read_pcap_info(const std::string &filePath, std::size_t &totalPakets) {
+    // libtins has a lot of overhead when just iterating through, so we use libpcap directly
+    char errbuf[PCAP_ERRBUF_SIZE];
+    pcap_t *pcap_handle = pcap_open_offline(filePath.c_str(), errbuf);
+    if (pcap_handle == nullptr) {
+        std::cerr << "ERROR: Could not open PCAP '" << filePath << "': " << errbuf << std::endl;
+        return false;
+    }
+
+    const u_char *packet;
+    pcap_pkthdr header;
+
+    packet = pcap_next(pcap_handle, &header);
+    if (packet == nullptr)
+    {
+        std::cerr << "ERROR: PCAP file is empty!" << std::endl;
+        pcap_close(pcap_handle);
+        return false;
+    }
+
+    // Extract first timestamp
+    stats.setTimestampFirstPacket(Tins::Timestamp(header.ts));
+
+    totalPakets = 0;
+    timeval lv;
+    while (packet != nullptr) {
+        totalPakets++;
+        // Extract last timestamp
+        lv = header.ts;
+        packet = pcap_next(pcap_handle, &header);
+    }
+
+    stats.setTimestampLastPacket(Tins::Timestamp(lv));
+
+    pcap_close(pcap_handle);
+    return true;
+}
+
 /**
  * Collect statistics of the loaded PCAP file. Calls for each packet the method process_packets.
  */
@@ -120,25 +158,20 @@ void pcap_processor::collect_statistics() {
     if (file_exists(filePath)) {
         std::cout << "Loading pcap..." << std::endl;
         FileSniffer sniffer(filePath);
-        FileSniffer snifferOverview(filePath);
 
         SnifferIterator i = sniffer.begin();
         std::chrono::microseconds currentPktTimestamp;
 
-        // Save timestamp of first packet
-        stats.setTimestampFirstPacket(i->timestamp());
+        // Read PCAP file info
+        std::size_t totalPackets = 0;
+        if (!read_pcap_info(filePath, totalPackets)) return;
 
-        int totalPackets = 0;
+        // choose a suitable time interval
         int timeIntervalCounter = 1;
         int timeIntervalsNum = 100;
         std::chrono::microseconds intervalStartTimestamp = stats.getTimestampFirstPacket();
         std::chrono::microseconds firstTimestamp = stats.getTimestampFirstPacket();
-
-        // An empty loop to know the capture duration, then choose a suitable time interval
-        SnifferIterator lastpkt;
-        for (SnifferIterator j = snifferOverview.begin(); j != snifferOverview.end(); ++j, ++totalPackets) {lastpkt = j;}
-
-        std::chrono::microseconds lastTimestamp = lastpkt->timestamp();
+        std::chrono::microseconds lastTimestamp = stats.getTimestampLastPacket();
         std::chrono::microseconds captureDuration = lastTimestamp - firstTimestamp;
         if(captureDuration.count()<=0){
             std::cout << "ERROR: PCAP file is empty!" << std::endl;

code_boost/src/cxx/pcap_processor.h

@@ -43,6 +43,8 @@ public:
 
     std::string merge_pcaps(const std::string pcap_path);
 
+    bool read_pcap_info(const std::string &filePath, std::size_t &totalPakets);
+
     void collect_statistics();
 
     void write_to_database(std::string database_path);