WIP: Added tbb

code_boost/src/CMakeLists.txt

@@ -81,7 +81,7 @@ SET_target_properties(sqlite3 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 ADD_LIBRARY(pcapreader SHARED ${SOURCE_FILES})
 # Libs pthread and dl are prerequisites of SQLiteCpp
-TARGET_LINK_LIBRARIES(pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl pcap)
+TARGET_LINK_LIBRARIES(pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl pcap tbb)
 
 ADD_LIBRARY(botnetcomm SHARED ${BOT_COMM_PROC_SOURCE})
 TARGET_LINK_LIBRARIES(botnetcomm ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})

code_boost/src/cxx/statistics.h

@@ -5,6 +5,7 @@
 #define CPP_PCAPREADER_STATISTICS_H
 
 #include <vector>
+#include "tbb/concurrent_unordered_map.h"
 #include <unordered_map>
 #include <unordered_set>
 #include <list>
@@ -649,64 +650,64 @@ private:
      * Data containers
      */
     // {IP Address, TTL value, count}
-    std::unordered_map<ipAddress_ttl, int> ttl_distribution;
+    tbb::concurrent_unordered_map<ipAddress_ttl, int> ttl_distribution;
 
     // {IP Address, MSS value, count}
-    std::unordered_map<ipAddress_mss, int> mss_distribution;
+    tbb::concurrent_unordered_map<ipAddress_mss, int> mss_distribution;
 
     // {IP Address, Win size, count}
-    std::unordered_map<ipAddress_win, int> win_distribution;
+    tbb::concurrent_unordered_map<ipAddress_win, int> win_distribution;
 
     // {IP Address, ToS value, count}
-    std::unordered_map<ipAddress_tos, int> tos_distribution;
+    tbb::concurrent_unordered_map<ipAddress_tos, int> tos_distribution;
 
     // {IP Address A, Port A, IP Address B, Port B,   #packets, packets timestamps, inter-arrival times,
     // average of inter-arrival times}
-    std::unordered_map<conv, entry_convStat> conv_statistics;
+    tbb::concurrent_unordered_map<conv, entry_convStat> conv_statistics;
 
     // {IP Address A, Port A, IP Address B, Port B,   comm_intervals, #packets, avg. pkt rate, avg. #packets per interval,
     // avg. time between intervals, avg. interval time, duration, packets timestamps, inter-arrivtal times, average of inter-arrival times}
     // Also stores conversation with only one exchanged message. In this case avgPktRate, minDelay, maxDelay and avgDelay are -1
-    std::unordered_map<convWithProt, entry_convStatExt> conv_statistics_extended;
+    tbb::concurrent_unordered_map<convWithProt, entry_convStatExt> conv_statistics_extended;
 
     // {Last timestamp in the interval, #packets, #bytes, source IP entropy, destination IP entropy,
     // source IP cumulative entropy, destination IP cumulative entropy, #payload, #incorrect TCP checksum,
     // #correct TCP checksum, #novel IP, #novel TTL, #novel Window Size, #novel ToS,#novel MSS}
-    std::unordered_map<std::string, entry_intervalStat> interval_statistics;
+    tbb::concurrent_unordered_map<std::string, entry_intervalStat> interval_statistics;
 
     // {TTL value, count}
-    std::unordered_map<int, int> ttl_values;
+    tbb::concurrent_unordered_map<int, int> ttl_values;
 
     // {Win size, count}
-    std::unordered_map<int, int> win_values;
+    tbb::concurrent_unordered_map<int, int> win_values;
 
     // {ToS, count}
-    std::unordered_map<int, int> tos_values;
+    tbb::concurrent_unordered_map<int, int> tos_values;
 
     // {MSS, count}
-    std::unordered_map<int, int> mss_values;
+    tbb::concurrent_unordered_map<int, int> mss_values;
 
     // {Port, count}
-    std::unordered_map<int, int> port_values;
+    tbb::concurrent_unordered_map<int, int> port_values;
 
 
     //{IP Address, contacted IP Addresses}
-    std::unordered_map<std::string, std::unordered_set<std::string>> contacted_ips;
+    tbb::concurrent_unordered_map<std::string, std::unordered_set<std::string>> contacted_ips;
 
     // {IP Address, Protocol,  #count, #Data transmitted in bytes}
-    std::unordered_map<ipAddress_protocol, entry_protocolStat> protocol_distribution;
+    tbb::concurrent_unordered_map<ipAddress_protocol, entry_protocolStat> protocol_distribution;
 
     // {IP Address,  #received packets, #sent packets, Data received in kbytes, Data sent in kbytes}
-    std::unordered_map<std::string, entry_ipStat> ip_statistics;
+    tbb::concurrent_unordered_map<std::string, entry_ipStat> ip_statistics;
 
     // {IP Address, in_out, Port Number,  #count, #Data transmitted in bytes}
-    std::unordered_map<ipAddress_inOut_port, entry_portStat> ip_ports;
+    tbb::concurrent_unordered_map<ipAddress_inOut_port, entry_portStat> ip_ports;
 
     // {IP Address, MAC Address}
-    std::unordered_map<std::string, std::string> ip_mac_mapping;
+    tbb::concurrent_unordered_map<std::string, std::string> ip_mac_mapping;
 
     // {Source MAC, Destination MAC, typeNumber, #count, #timestamp of last occurrence}
-    std::unordered_map<unrecognized_PDU, unrecognized_PDU_stat> unrecognized_PDUs;
+    tbb::concurrent_unordered_map<unrecognized_PDU, unrecognized_PDU_stat> unrecognized_PDUs;
 };
 
 

code_boost/src/cxx/statistics_db.cpp

@@ -27,7 +27,7 @@ statistics_db::statistics_db(std::string database_path) {
  * Writes the IP statistics into the database.
  * @param ipStatistics The IP statistics from class statistics.
  */
-void statistics_db::writeStatisticsIP(const std::unordered_map<std::string, entry_ipStat> &ipStatistics) {
+void statistics_db::writeStatisticsIP(const tbb::concurrent_unordered_map<std::string, entry_ipStat> &ipStatistics) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_statistics");
         SQLite::Transaction transaction(*db);
@@ -68,7 +68,7 @@ void statistics_db::writeStatisticsIP(const std::unordered_map<std::string, entr
  * @param ipStatistics The IP statistics from class statistics. Degree Statistics are supposed to be integrated into the ip_statistics table later on,
  *        therefore they use the same parameter. But for now they are inserted into their own table.
  */
-void statistics_db::writeStatisticsDegree(const std::unordered_map<std::string, entry_ipStat> &ipStatistics){
+void statistics_db::writeStatisticsDegree(const tbb::concurrent_unordered_map<std::string, entry_ipStat> &ipStatistics){
     try {
         db->exec("DROP TABLE IF EXISTS ip_degrees");
         SQLite::Transaction transaction(*db);
@@ -100,7 +100,7 @@ void statistics_db::writeStatisticsDegree(const std::unordered_map<std::string,
  * Writes the TTL distribution into the database.
  * @param ttlDistribution The TTL distribution from class statistics.
  */
-void statistics_db::writeStatisticsTTL(const std::unordered_map<ipAddress_ttl, int> &ttlDistribution) {
+void statistics_db::writeStatisticsTTL(const tbb::concurrent_unordered_map<ipAddress_ttl, int> &ttlDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_ttl");
         SQLite::Transaction transaction(*db);
@@ -131,7 +131,7 @@ void statistics_db::writeStatisticsTTL(const std::unordered_map<ipAddress_ttl, i
  * Writes the MSS distribution into the database.
  * @param mssDistribution The MSS distribution from class statistics.
  */
-void statistics_db::writeStatisticsMSS(const std::unordered_map<ipAddress_mss, int> &mssDistribution) {
+void statistics_db::writeStatisticsMSS(const tbb::concurrent_unordered_map<ipAddress_mss, int> &mssDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS tcp_mss");
         SQLite::Transaction transaction(*db);
@@ -162,7 +162,7 @@ void statistics_db::writeStatisticsMSS(const std::unordered_map<ipAddress_mss, i
  * Writes the ToS distribution into the database.
  * @param tosDistribution The ToS distribution from class statistics.
  */
-void statistics_db::writeStatisticsToS(const std::unordered_map<ipAddress_tos, int> &tosDistribution) {
+void statistics_db::writeStatisticsToS(const tbb::concurrent_unordered_map<ipAddress_tos, int> &tosDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_tos");
         SQLite::Transaction transaction(*db);
@@ -192,7 +192,7 @@ void statistics_db::writeStatisticsToS(const std::unordered_map<ipAddress_tos, i
  * Writes the window size distribution into the database.
  * @param winDistribution The window size distribution from class statistics.
  */
-void statistics_db::writeStatisticsWin(const std::unordered_map<ipAddress_win, int> &winDistribution) {
+void statistics_db::writeStatisticsWin(const tbb::concurrent_unordered_map<ipAddress_win, int> &winDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS tcp_win");
         SQLite::Transaction transaction(*db);
@@ -223,7 +223,7 @@ void statistics_db::writeStatisticsWin(const std::unordered_map<ipAddress_win, i
  * Writes the protocol distribution into the database.
  * @param protocolDistribution The protocol distribution from class statistics.
  */
-void statistics_db::writeStatisticsProtocols(const std::unordered_map<ipAddress_protocol, entry_protocolStat> &protocolDistribution) {
+void statistics_db::writeStatisticsProtocols(const tbb::concurrent_unordered_map<ipAddress_protocol, entry_protocolStat> &protocolDistribution) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_protocols");
         SQLite::Transaction transaction(*db);
@@ -255,7 +255,7 @@ void statistics_db::writeStatisticsProtocols(const std::unordered_map<ipAddress_
  * Writes the port statistics into the database.
  * @param portsStatistics The ports statistics from class statistics.
  */
-void statistics_db::writeStatisticsPorts(const std::unordered_map<ipAddress_inOut_port, entry_portStat> &portsStatistics) {
+void statistics_db::writeStatisticsPorts(const tbb::concurrent_unordered_map<ipAddress_inOut_port, entry_portStat> &portsStatistics) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_ports");
         SQLite::Transaction transaction(*db);
@@ -300,7 +300,7 @@ void statistics_db::writeStatisticsPorts(const std::unordered_map<ipAddress_inOu
  *  Writes the IP address -> MAC address mapping into the database.
  * @param IpMacStatistics The IP address -> MAC address mapping from class statistics.
  */
-void statistics_db::writeStatisticsIpMac(const std::unordered_map<std::string, std::string> &IpMacStatistics) {
+void statistics_db::writeStatisticsIpMac(const tbb::concurrent_unordered_map<std::string, std::string> &IpMacStatistics) {
     try {
         db->exec("DROP TABLE IF EXISTS ip_mac");
         SQLite::Transaction transaction(*db);
@@ -378,7 +378,7 @@ void statistics_db::writeStatisticsFile(int packetCount, float captureDuration,
  * Writes the conversation statistics into the database.
  * @param convStatistics The conversation from class statistics.
  */
-void statistics_db::writeStatisticsConv(std::unordered_map<conv, entry_convStat> &convStatistics){
+void statistics_db::writeStatisticsConv(tbb::concurrent_unordered_map<conv, entry_convStat> &convStatistics){
     try {
         db->exec("DROP TABLE IF EXISTS conv_statistics");
         SQLite::Transaction transaction(*db);
@@ -444,7 +444,7 @@ void statistics_db::writeStatisticsConv(std::unordered_map<conv, entry_convStat>
  * Writes the extended statistics for every conversation into the database.
  * @param conv_statistics_extended The extended conversation statistics from class statistics.
  */
-void statistics_db::writeStatisticsConvExt(std::unordered_map<convWithProt, entry_convStatExt> &conv_statistics_extended){
+void statistics_db::writeStatisticsConvExt(tbb::concurrent_unordered_map<convWithProt, entry_convStatExt> &conv_statistics_extended){
     try {
         db->exec("DROP TABLE IF EXISTS conv_statistics_extended");
         SQLite::Transaction transaction(*db);
@@ -538,7 +538,7 @@ void statistics_db::writeStatisticsConvExt(std::unordered_map<convWithProt, entr
  * Writes the interval statistics into the database.
  * @param intervalStatistics The interval entries from class statistics.
  */
-void statistics_db::writeStatisticsInterval(const std::unordered_map<std::string, entry_intervalStat> &intervalStatistics){
+void statistics_db::writeStatisticsInterval(const tbb::concurrent_unordered_map<std::string, entry_intervalStat> &intervalStatistics){
     try {        
         db->exec("DROP TABLE IF EXISTS interval_statistics");
         SQLite::Transaction transaction(*db);
@@ -704,7 +704,7 @@ bool statistics_db::pathExists(std::string path)
  * Writes the unrecognized PDUs into the database.
  * @param unrecognized_PDUs The unrecognized PDUs from class statistics.
  */
-void statistics_db::writeStatisticsUnrecognizedPDUs(const std::unordered_map<unrecognized_PDU, unrecognized_PDU_stat>
+void statistics_db::writeStatisticsUnrecognizedPDUs(const tbb::concurrent_unordered_map<unrecognized_PDU, unrecognized_PDU_stat>
                                                     &unrecognized_PDUs) {
     try {
         db->exec("DROP TABLE IF EXISTS unrecognized_pdus");

code_boost/src/cxx/statistics_db.h

@@ -27,34 +27,34 @@ public:
     /*
      * Methods for writing values into database
      */
-    void writeStatisticsIP(const std::unordered_map<std::string, entry_ipStat> &ipStatistics);
+    void writeStatisticsIP(const tbb::concurrent_unordered_map<std::string, entry_ipStat> &ipStatistics);
 
-    void writeStatisticsDegree(const std::unordered_map<std::string, entry_ipStat> &ipStatistics);
+    void writeStatisticsDegree(const tbb::concurrent_unordered_map<std::string, entry_ipStat> &ipStatistics);
 
-    void writeStatisticsTTL(const std::unordered_map<ipAddress_ttl, int> &ttlDistribution);
+    void writeStatisticsTTL(const tbb::concurrent_unordered_map<ipAddress_ttl, int> &ttlDistribution);
 
-    void writeStatisticsMSS(const std::unordered_map<ipAddress_mss, int> &mssDistribution);
+    void writeStatisticsMSS(const tbb::concurrent_unordered_map<ipAddress_mss, int> &mssDistribution);
 
-    void writeStatisticsToS(const std::unordered_map<ipAddress_tos, int> &tosDistribution);
+    void writeStatisticsToS(const tbb::concurrent_unordered_map<ipAddress_tos, int> &tosDistribution);
 
-    void writeStatisticsWin(const std::unordered_map<ipAddress_win, int> &winDistribution);
+    void writeStatisticsWin(const tbb::concurrent_unordered_map<ipAddress_win, int> &winDistribution);
 
-    void writeStatisticsProtocols(const std::unordered_map<ipAddress_protocol, entry_protocolStat> &protocolDistribution);
+    void writeStatisticsProtocols(const tbb::concurrent_unordered_map<ipAddress_protocol, entry_protocolStat> &protocolDistribution);
 
-    void writeStatisticsPorts(const std::unordered_map<ipAddress_inOut_port, entry_portStat> &portsStatistics);
+    void writeStatisticsPorts(const tbb::concurrent_unordered_map<ipAddress_inOut_port, entry_portStat> &portsStatistics);
 
-    void writeStatisticsIpMac(const std::unordered_map<std::string, std::string> &IpMacStatistics);
+    void writeStatisticsIpMac(const tbb::concurrent_unordered_map<std::string, std::string> &IpMacStatistics);
 
     void writeStatisticsFile(int packetCount, float captureDuration, std::string timestampFirstPkt,
                              std::string timestampLastPkt, float avgPacketRate, float avgPacketSize,
                              float avgPacketsSentPerHost, float avgBandwidthIn, float avgBandwidthOut,
                              bool doExtraTests);
 
-    void writeStatisticsConv(std::unordered_map<conv, entry_convStat> &convStatistics);
+    void writeStatisticsConv(tbb::concurrent_unordered_map<conv, entry_convStat> &convStatistics);
 
-    void writeStatisticsConvExt(std::unordered_map<convWithProt, entry_convStatExt> &conv_statistics_extended);
+    void writeStatisticsConvExt(tbb::concurrent_unordered_map<convWithProt, entry_convStatExt> &conv_statistics_extended);
 
-    void writeStatisticsInterval(const std::unordered_map<std::string, entry_intervalStat> &intervalStatistics);
+    void writeStatisticsInterval(const tbb::concurrent_unordered_map<std::string, entry_intervalStat> &intervalStatistics);
 
     void writeDbVersion();
 
@@ -64,7 +64,7 @@ public:
 
     bool pathExists(std::string path);
 
-    void writeStatisticsUnrecognizedPDUs(const std::unordered_map<unrecognized_PDU, unrecognized_PDU_stat> &unrecognized_PDUs);
+    void writeStatisticsUnrecognizedPDUs(const tbb::concurrent_unordered_map<unrecognized_PDU, unrecognized_PDU_stat> &unrecognized_PDUs);
 
 
 private: