build.sh

@@ -16,8 +16,15 @@ else
 fi
 cmake ..
 
+# Make sure we're able to get the number of cores
+if [ $(uname) = 'Darwin' ]; then
+    NUMCORES=$(sysctl -n hw.logicalcpu)
+else
+    NUMCORES=$(nproc)
+fi
+
 if [ -f Makefile ]; then
-    make
+    make -j$NUMCORES
 else
     echo "Error: 'cmake' did not finish successfully."
     exit

code/Attack/DDoSAttack.py

@@ -55,7 +55,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
         # attacker configuration
         num_attackers = rnd.randint(1, 16)
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
 
         self.add_param_value(atkParam.Parameter.IP_SOURCE,
                              self.generate_random_ipv4_address(most_used_ip_class, num_attackers))
@@ -82,7 +82,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
         if (num_attackers is not None) and (num_attackers is not 0):
             # user supplied atkParam.Parameter.NUMBER_ATTACKERS
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             # Create random attackers based on user input atkParam.Parameter.NUMBER_ATTACKERS
             ip_source_list = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
             mac_source_list = self.generate_random_mac_address(num_attackers)
@@ -164,14 +164,14 @@ class DDoSAttack(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
             destination_win_value = destination_win_prob_dict.random()
         else:
-            destination_win_value = self.statistics.process_db_query("most_used(winSize)")
+            destination_win_value = self.statistics.get_most_used_win_size()
 
         destination_win_value = Util.handle_most_used_outputs(destination_win_value)
 
         # MSS that was used by IP destination in background traffic
         mss_dst = self.statistics.get_most_used_mss(ip_destination)
         if mss_dst is None:
-            mss_dst = self.statistics.process_db_query("most_used(mssValue)")
+            mss_dst = self.statistics.get_most_used_mss_value()
 
         mss_dst = Util.handle_most_used_outputs(mss_dst)
 

code/Attack/EternalBlueExploit.py

@@ -106,7 +106,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -114,7 +114,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -132,7 +132,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
 
         # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-        mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+        mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         if not mss_value:
             mss_value = 1465
 

code/Attack/FTPWinaXeExploit.py

@@ -50,7 +50,7 @@ class FTPWinaXeExploit(BaseAttack.BaseAttack):
         most_used_ip_address = self.statistics.get_most_used_ip_address()
 
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
         attacker_ip = self.generate_random_ipv4_address(most_used_ip_class)
         self.add_param_value(atkParam.Parameter.IP_DESTINATION, attacker_ip)
         self.add_param_value(atkParam.Parameter.MAC_DESTINATION, self.generate_random_mac_address())
@@ -98,7 +98,7 @@ class FTPWinaXeExploit(BaseAttack.BaseAttack):
         # Create random victim if specified
         if self.get_param_value(atkParam.Parameter.IP_SOURCE_RANDOMIZE):
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             ip_victim = self.generate_random_ipv4_address(most_used_ip_class, 1)
             mac_victim = self.generate_random_mac_address()
 

code/Attack/JoomlaRegPrivExploit.py

@@ -107,7 +107,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -115,7 +115,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Inject Joomla_registration_privesc
         # Read joomla_registration_privesc pcap file

code/Attack/MS17ScanAttack.py

@@ -104,7 +104,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -112,7 +112,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -130,7 +130,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
 
         # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-        mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+        mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         if not mss_value:
             mss_value = 1465
 

code/Attack/PortscanAttack.py

@@ -139,14 +139,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_mss_prob_dict = lea.Lea.fromValFreqsDict(source_mss_dist)
             source_mss_value = source_mss_prob_dict.random()
         else:
-            source_mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+            source_mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         destination_mss_dist = self.statistics.get_mss_distribution(ip_destination)
         if len(destination_mss_dist) > 0:
             destination_mss_prob_dict = lea.Lea.fromValFreqsDict(destination_mss_dist)
             destination_mss_value = destination_mss_prob_dict.random()
         else:
-            destination_mss_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(mssValue)"))
+            destination_mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
 
         # Set TTL based on TTL distribution of IP address
         source_ttl_dist = self.statistics.get_ttl_distribution(ip_source)
@@ -154,14 +153,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
             destination_ttl_prob_dict = lea.Lea.fromValFreqsDict(destination_ttl_dist)
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
-            destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+            destination_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -169,14 +167,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_win_prob_dict = lea.Lea.fromValFreqsDict(source_win_dist)
             source_win_value = source_win_prob_dict.random()
         else:
-            source_win_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(winSize)"))
+            source_win_value = Util.handle_most_used_outputs(self.statistics.get_most_used_win_size())
         destination_win_dist = self.statistics.get_win_distribution(ip_destination)
         if len(destination_win_dist) > 0:
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
             destination_win_value = destination_win_prob_dict.random()
         else:
-            destination_win_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(winSize)"))
+            destination_win_value = Util.handle_most_used_outputs(self.statistics.get_most_used_win_size())
 
         min_delay, max_delay = self.get_reply_delay(ip_destination)
 

code/Attack/SMBLorisAttack.py

@@ -48,7 +48,7 @@ class SMBLorisAttack(BaseAttack.BaseAttack):
         most_used_ip_address = self.statistics.get_most_used_ip_address()
 
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
         num_attackers = rnd.randint(1, 16)
         source_ip = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
 
@@ -89,7 +89,7 @@ class SMBLorisAttack(BaseAttack.BaseAttack):
         # user supplied atkParam.Parameter.NUMBER_ATTACKERS
         if (num_attackers is not None) and (num_attackers is not 0):
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             # Create random attackers based on user input atkParam.Parameter.NUMBER_ATTACKERS
             ip_source = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
             mac_source = self.generate_random_mac_address(num_attackers)

code/Attack/SQLiAttack.py

@@ -111,7 +111,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -119,7 +119,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Inject SQLi Attack
         # Read SQLi Attack pcap file

code/Core/Controller.py

@@ -8,6 +8,7 @@ import Core.LabelManager as LabelManager
 import Core.Statistics as Statistics
 import ID2TLib.PcapFile as PcapFile
 import ID2TLib.Utility as Util
+import Core.StatsDatabase as StatsDB
 
 
 class Controller:
@@ -176,7 +177,7 @@ class Controller:
             print()
         elif param == "least_used":
             print("least_used can be used as a selector for the following attributes:")
-            print("ipAddress | macAddress | portNumber | protocolName | ttlValue")
+            print("ipAddress | macAddress | portNumber | protocolName | ttlValue | mssValue | winSize | ipClass")
             print()
         elif param == "avg":
             print("avg can be used as a selector for the following attributes:")
@@ -184,7 +185,7 @@ class Controller:
             print()
         elif param == "all":
             print("all can be used as a selector for the following attributes:")
-            print("ipAddress | ttlValue | mss | macAddress | portNumber | protocolName")
+            print("ipAddress | ttlValue | mss | macAddress | portNumber | protocolName | winSize | ipClass")
             print()
         elif param in ["random", "first", "last"]:
             print("No additional info available for this keyword.")
@@ -197,6 +198,14 @@ class Controller:
                   "macAddress | ttlValue | ttlCount | portDirection | portNumber | portCount | protocolCount\n"
                   "protocolName")
             print()
+            print("The following operators can be used:")
+            print("<= | < | = | >= | > | in")
+            print()
+            print("A value can either be a simple values, a list of simple values separated by commas and enclosed "
+                  "in [] brackets, or another query.")
+            print()
+            print("When VALUE is a list (or a query returning a list), the usage of the 'in' operator is mandatory!")
+            print()
             print("See 'help examples;' for usage examples.")
             print()
         elif param == "macaddress":
@@ -205,6 +214,8 @@ class Controller:
             print("The following parameters can be specified:")
             print("ipAddress")
             print()
+            print("See 'help ipAddress' for information on valid operators and values.")
+            print()
             print("See 'help examples;' for usage examples.")
             print()
         elif param == "examples":
@@ -218,6 +229,8 @@ class Controller:
             print("\tSELECT avg(ttlValue) from ip_ttl;")
             print("Get a random IP address from all addresses that sent and received at least 10 packets:")
             print("\trandom(ipAddress(pktsSent > 10, pktsReceived > 10));")
+            print("Get the IP addresses used with one of the MAC addresses in a list:")
+            print("\tipAddress(macAddress in [08:00:27:a3:83:43, 52:54:00:12:35:02]);")
             print()
         else:
             print("Unknown keyword '" + param + "', try 'help;' to get a list of allowed keywords'")
@@ -291,6 +304,9 @@ class Controller:
                         for i in range(1, e.col):
                             sys.stderr.write(" ")
                         sys.stderr.write("^\n\n")
+                    except StatsDB.QueryExecutionException as e:
+                        sys.stderr.write("An error occured: ")
+                        sys.stderr.write(e.args[0] + "\n")
                 buffer = ""
 
         readline.set_history_length(1000)

code/Core/QueryParser.py

@@ -3,24 +3,75 @@ import pyparsing as pp
 
 class QueryParser:
     def __init__(self):
+        """
+        Constructs a parser for all named queries using PyParsing.
+        """
         extractor = pp.Keyword("random") ^ pp.Keyword("first") ^ pp.Keyword("last")
-        selector = pp.Keyword("most_used") ^ pp.Keyword("least_used") ^ pp.Keyword("avg") ^ pp.Keyword("all")
-        attribute = pp.Keyword("ipaddress") ^ pp.Keyword("macaddress") ^ pp.Keyword("portnumber") ^ pp.Keyword("protocolname") ^ pp.Keyword("ttlvalue") ^ pp.Keyword("mssvalue") ^ pp.Keyword("winsize") ^ pp.Keyword("ipclass") ^ pp.Keyword("pktssent") ^ pp.Keyword("pktsreceived") ^ pp.Keyword("mss") ^ pp.Keyword("kbytesreceived") ^ pp.Keyword("kbytessent")
-        simple_selector_query = selector + pp.Suppress("(") + attribute + pp.Suppress(")")
 
-        param_selectors = pp.Keyword("ipaddress").setParseAction(pp.replaceWith("ipaddress_param")) ^ pp.Keyword("macaddress").setParseAction(pp.replaceWith("macaddress_param"))
-        operators = pp.Literal("=") ^ pp.Literal("<=") ^ pp.Literal("<") ^ pp.Literal(">=") ^ pp.Literal(">")
+        # Valid selectors - except "avg", because not all attributes can be combined with it
+        selector_no_avg = pp.Keyword("most_used") ^ pp.Keyword("least_used") ^ pp.Keyword("all")
+
+        # All attributes that cannot be combined with "avg"
+        attributes_no_avg = pp.Keyword("ipaddress") ^ pp.Keyword("macaddress") ^ pp.Keyword("portnumber") ^\
+                            pp.Keyword("protocolname") ^ pp.Keyword("winsize") ^ pp.Keyword("ipclass")
+
+        # All attributes that can be combined with "avg"
+        attributes_avg = pp.Keyword("ttlvalue") ^ pp.Keyword("mssvalue") ^\
+                         pp.Keyword("pktssent") ^ pp.Keyword("pktsreceived") ^ pp.Keyword("mss") ^\
+                         pp.Keyword("kbytesreceived") ^ pp.Keyword("kbytessent")
+
+        # Collection of all attributes for simpler specification
+        attributes_all = attributes_no_avg ^ attributes_avg
+
+        # Simple selector + attribute query, only allowing "avg" with compatible attributes
+        simple_selector_query = (selector_no_avg + pp.Suppress("(") + attributes_all + pp.Suppress(")")) ^\
+                                (pp.Keyword("avg") + pp.Suppress("(") + attributes_avg + pp.Suppress(")"))
+
+        # Selectors for parameterized queries - they are replaced in the result to avoid ambiguity
+        param_selectors = pp.Keyword("ipaddress").setParseAction(pp.replaceWith("ipaddress_param")) ^\
+                          pp.Keyword("macaddress").setParseAction(pp.replaceWith("macaddress_param"))
+
+        # All operators allowed in parameterized queries
+        operators = pp.Literal("<=") ^ pp.Literal("<") ^ pp.Literal("=") ^\
+                    pp.Literal(">=") ^ pp.Literal(">") ^ pp.CaselessLiteral("in")
+
+        # Placeholder for nesting in parameterized queries
         expr = pp.Forward()
-        comparison = pp.Group(attribute + operators + (pp.Word(pp.alphanums + ".:") ^ expr))
+
+        # Simple values for comparisons inside a parameterized query can be alphanumeric plus dot and colon
+        simple_value = pp.Word(pp.alphanums + ".:")
+
+        # Values in parameterized queries can either be simple values, or a list of them.
+        # If it's a list, we insert a "list"-token to be able to distinguish it
+        parameterized_value = simple_value ^\
+                              (pp.Suppress("[") + pp.Group(pp.Empty().addParseAction(pp.replaceWith('list')) +
+                               pp.delimitedList(simple_value)) + pp.Suppress("]"))
+
+        # One "attribute-operator-value" triplet for parameterized queries
+        comparison = pp.Group(attributes_all + operators + (parameterized_value ^ expr))
+
+        # A full parameterized query, consisting of a parameterized selector and a comma-separated list of comparisons
         parameterized_query = param_selectors + pp.Suppress("(") + pp.Group(pp.delimitedList(comparison)) + pp.Suppress(")")
-        # parameterized_query = param_selectors + pp.Suppress("(") + comparison + pp.Suppress(")")
 
+        # Combination of simple and parameterized queries
         all_selector_queries = (simple_selector_query ^ parameterized_query)
+
+        # All queries can be combined with an extractor
         extractor_selector_query = extractor + pp.Suppress("(") + all_selector_queries + pp.Suppress(")")
 
+        # Queries can be used with an extractor or without
         named_query = (extractor_selector_query ^ all_selector_queries)
+
+        # The placeholder can be replaced with any query
         expr << pp.Group(named_query)
+
+        # Make sure all queries end with a semicolon, and we're done
         self.full_query = named_query + pp.Suppress(";")
 
-    def parse_query(self, querystring):
+    def parse_query(self, querystring: str) -> pp.ParseResults:
+        """
+        Parses the passed query with a pre-constructed parser.
+        :param querystring: The named query to be executed
+        :return: A ParseResults-object, which essentially is a list of tokens
+        """
         return self.full_query.parseString(querystring)

code/Core/Statistics.py

@@ -544,7 +544,7 @@ class Statistics:
         :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of
         randomly chosen IP addresses
         """
-        ip_address_list = self.process_db_query("all(ipAddress)")
+        ip_address_list = self.process_db_query("SELECT ipAddress from ip_statistics ORDER BY ipAddress ASC")
         if count == 1:
             return random.choice(ip_address_list)
         else:
@@ -560,13 +560,45 @@ class Statistics:
         :param mac_address: the MAC address of which the IP shall be returned, if existing in DB
         :return: the IP address used in the dataset by a given MAC address
         """
-        return self.process_db_query('ipAddress(macAddress=' + mac_address + ")")
+        return self.process_db_query("SELECT DISTINCT ipAddress FROM ip_mac WHERE macAddress = '" + mac_address + "'")
 
     def get_mac_address(self, ip_address: str):
         """
         :return: The MAC address used in the dataset for the given IP address.
         """
-        return self.process_db_query('macAddress(ipAddress=' + ip_address + ")")
+        return self.process_db_query("SELECT DISTINCT macAddress from ip_mac WHERE ipAddress = '" + ip_address + "'")
+
+    def get_most_used_ttl_value(self):
+        """
+        :return: The most used TTL value.
+        """
+        return self.process_db_query("SELECT ttlValue FROM (SELECT ttlValue, SUM(ttlCount) as occ FROM ip_ttl GROUP BY "
+                                     "ttlValue) WHERE occ=(SELECT SUM(ttlCount) as occ FROM ip_ttl GROUP BY ttlValue "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY ttlValue ASC")
+
+    def get_most_used_ip_class(self):
+        """
+        :return: The most used IP class.
+        """
+        return self.process_db_query("SELECT ipClass FROM (SELECT ipClass, COUNT(*) as occ from ip_statistics GROUP BY "
+                                     "ipClass ORDER BY occ DESC) WHERE occ=(SELECT COUNT(*) as occ from ip_statistics "
+                                     "GROUP BY ipClass ORDER BY occ DESC LIMIT 1) ORDER BY ipClass ASC")
+
+    def get_most_used_win_size(self):
+        """
+        :return: The most used window size.
+        """
+        return self.process_db_query("SELECT winSize FROM (SELECT winSize, SUM(winCount) as occ FROM tcp_win GROUP BY "
+                                     "winSize) WHERE occ=(SELECT SUM(winCount) as occ FROM tcp_win GROUP BY winSize "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY winSize ASC")
+
+    def get_most_used_mss_value(self):
+        """
+        :return: The most used mss value.
+        """
+        return self.process_db_query("SELECT mssValue FROM (SELECT mssValue, SUM(mssCount) as occ FROM tcp_mss GROUP BY"
+                                     " mssValue) WHERE occ=(SELECT SUM(mssCount) as occ FROM tcp_mss GROUP BY mssValue "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY mssValue ASC")
 
     def get_most_used_mss(self, ip_address: str):
         """
@@ -1061,6 +1093,8 @@ class Statistics:
         print("\nNew database has been generated, printing statistics summary... ")
         total_packet_count = self.get_packet_count()
         pdu_count = self.process_db_query("SELECT SUM(pktCount) FROM unrecognized_pdus")
+        if pdu_count is None:
+            pdu_count = 0
         pdu_share = pdu_count / total_packet_count * 100
         last_pdu_timestamp = self.process_db_query(
             "SELECT MAX(timestampLastOccurrence) FROM unrecognized_pdus")

code/Core/StatsDatabase.py

@@ -25,6 +25,10 @@ def dict_gen(curs: sqlite3.Cursor):
             yield dict(zip(field_names, row))
 
 
+class QueryExecutionException(Exception):
+    pass
+
+
 class StatsDatabase:
     def __init__(self, db_path: str):
         """
@@ -168,18 +172,48 @@ class StatsDatabase:
         field_types = self.get_field_types('ip_mac', 'ip_ttl', 'ip_ports', 'ip_protocols', 'ip_statistics', 'ip_mac')
         conditions = []
         for key, op, value in param_op_val:
+            # Check whether the value is not a simple value, but another query (or list)
             if isinstance(value, pp.ParseResults):
-                # If we have another query instead of a direct value, execute and replace it
-                value = self._execute_query_list(value)[0][0]
+                if value[0] == "list":
+                    # We have a list, cut the token off and use the remaining elements
+                    value = value[1:]
+
+                    # Lists can only be used with "in"
+                    if op is not "in":
+                        raise QueryExecutionException("List values require the usage of the 'in' operator!")
+                else:
+                    # If we have another query instead of a direct value, execute and replace it
+                    rvalue = self._execute_query_list(value)
+
+                    # Do we have a comparison operator with a multiple-result query?
+                    if op is not "in" and value[0] in ['most_used', 'least_used', 'all', 'ipaddress_param',
+                                                       'macaddress_param']:
+                        raise QueryExecutionException("The extractor '" + value[0] +
+                                                      "' may return more than one result!")
+
+                    # Make value contain a simple list with the results of the query
+                    value = map(lambda x: str(x[0]), rvalue)
+            else:
+                # Make sure value is a list now to simplify handling
+                value = [value]
+
             # this makes sure that TEXT fields are queried by strings,
             # e.g. ipAddress=192.168.178.1 --is-converted-to--> ipAddress='192.168.178.1'
             if field_types.get(key) == 'TEXT':
-                if not str(value).startswith("'") and not str(value).startswith('"'):
-                    value = "'" + value + "'"
+                def ensure_string(x):
+                    if not str(x).startswith("'") and not str(x).startswith('"'):
+                        return "'" + x + "'"
+                    else:
+                        return x
+                value = map(ensure_string, value)
+
+            # If we have more than one value, join them together, separated by commas
+            value = ",".join(map(str, value))
+
             # this replacement is required to remove ambiguity in SQL query
             if key == 'ipAddress':
                 key = 'ip_mac.ipAddress'
-            conditions.append(key + op + str(value))
+            conditions.append(key + " " + op + " (" + str(value) + ")")
 
         where_clause = " AND ".join(conditions)
         query += where_clause
@@ -230,6 +264,9 @@ class StatsDatabase:
         "least_used.winsize": "SELECT winSize FROM (SELECT winSize, SUM(winCount) as occ FROM tcp_win GROUP BY "
                               "winSize) WHERE occ=(SELECT SUM(winCount) as occ FROM tcp_win GROUP BY winSize "
                               "ORDER BY occ ASC LIMIT 1) ORDER BY winSize ASC",
+        "least_used.ipclass": "SELECT ipClass FROM (SELECT ipClass, COUNT(*) as occ from ip_statistics GROUP BY "
+                             "ipClass ORDER BY occ DESC) WHERE occ=(SELECT COUNT(*) as occ from ip_statistics "
+                             "GROUP BY ipClass ORDER BY occ ASC LIMIT 1) ORDER BY ipClass ASC",
         "avg.pktsreceived": "SELECT avg(pktsReceived) from ip_statistics",
         "avg.pktssent": "SELECT avg(pktsSent) from ip_statistics",
         "avg.kbytesreceived": "SELECT avg(kbytesReceived) from ip_statistics",
@@ -241,27 +278,32 @@ class StatsDatabase:
         "all.mss": "SELECT DISTINCT mssValue from tcp_mss ORDER BY mssValue ASC",
         "all.macaddress": "SELECT DISTINCT macAddress from ip_mac ORDER BY macAddress ASC",
         "all.portnumber": "SELECT DISTINCT portNumber from ip_ports ORDER BY portNumber ASC",
-        "all.protocolname": "SELECT DISTINCT protocolName from ip_protocols ORDER BY protocolName ASC"}
+        "all.protocolname": "SELECT DISTINCT protocolName from ip_protocols ORDER BY protocolName ASC",
+        "all.winsize": "SELECT DISTINCT winSize FROM tcp_win ORDER BY winSize ASC",
+        "all.ipclass": "SELECT DISTINCT ipClass FROM ip_statistics ORDER BY ipClass ASC"}
 
     def _execute_query_list(self, query_list):
         """
         Recursively executes a list of named queries. They are of the following form:
-        ['macaddress_param', [['ipaddress', '=', ['most_used', 'ipaddress']]]]
+        ['macaddress_param', [['ipaddress', 'in', ['most_used', 'ipaddress']]]]
         :param query_list: The query statement list obtained from the query parser
         :return: The result of the query (either a single result or a list).
         """
         if query_list[0] == "random":
-            return rnd.choice(self._execute_query_list(query_list[1:]))
+            return [rnd.choice(self._execute_query_list(query_list[1:]))]
         elif query_list[0] == "first":
-            return self._execute_query_list(query_list[1:])[0]
+            return [self._execute_query_list(query_list[1:])[0]]
         elif query_list[0] == "last":
-            return self._execute_query_list(query_list[1:])[-1]
+            return [self._execute_query_list(query_list[1:])[-1]]
         elif query_list[0] == "macaddress_param":
             return self.named_query_parameterized("macaddress", query_list[1])
         elif query_list[0] == "ipaddress_param":
             return self.named_query_parameterized("ipaddress", query_list[1])
         else:
             query = self.named_queries.get(query_list[0] + "." + query_list[1])
+            if query is None:
+                raise QueryExecutionException("The requested query '" + query_list[0] + "(" + query_list[1] +
+                                              ")' was not found in the internal query list!")
             self.cursor.execute(str(query))
             last_result = self.cursor.fetchall()
             return last_result

code/Test/efficiency_testing.py

@@ -5,65 +5,50 @@ import Test.ID2TAttackTest as Test
 
 
 class EfficiencyTests(Test.ID2TAttackTest):
-    def test_SMBLoris_10_000(self):
-        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=30', 'packets.per-second=8.0']],
-                                      time_limit=15, factor=10000)
+    def test_SMBLoris(self):
+        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=4', 'packets.per-second=8.0']],
+                                      time_limit=1.5, factor=1000)
 
-    def test_SMBLoris_100_000(self):
-        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=30', 'packets.per-second=98']],
-                                      time_limit=150, factor=100000)
-
-    def test_SMBScan_10_000(self):
+    def test_SMBScan(self):
         self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1',
-                                        'ip.dst=192.168.178.10-192.168.197.145']], time_limit=15, factor=10000)
-
-    def test_SMBScan_100_000(self):
-        self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1', 'ip.dst=192.168.0.1-192.168.195.76']],
-                                      time_limit=150, factor=100000)
+                                        'ip.dst=192.168.178.10-192.168.179.253']], time_limit=1.5, factor=1000)
 
-    def test_SMBScan_hosting_10_000(self):
+    def test_SMBScan_hosting(self):
         self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1',
-                                        'ip.dst=192.168.178.10-192.168.181.241',
-                                        'hosting.ip=192.168.178.10-192.168.181.241']], time_limit=15, factor=10000)
-
-    def test_SMBScan_hosting_100_000(self):
-        self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1', 'ip.dst=192.168.178.10-192.168.217.25',
-                                        'hosting.ip=192.168.178.10-192.168.217.25']], time_limit=150, factor=100000)
+                                        'ip.dst=192.168.178.10-192.168.178.109',
+                                        'hosting.ip=192.168.178.10-192.168.178.109']], time_limit=1.5, factor=1000)
 
     @mock.patch('ID2TLib.Utility.get_rnd_bytes', side_effect=Lib.get_bytes)
     @mock.patch('ID2TLib.Utility.get_rnd_x86_nop', side_effect=Lib.get_x86_nop)
     def test_FTPExploit(self, mock_get_rnd_x86_nop, mock_get_rnd_bytes):
         self.temporal_efficiency_test([['FTPWinaXeExploit', 'ip.src=192.168.178.1', 'ip.dst=192.168.178.10']],
-                                      time_limit=15, factor=10000)
+                                      time_limit=1.5, factor=1000)
 
-    def test_PortscanAttack_open_10_000(self):
-        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=80']], time_limit=15,
-                                      factor=10000)
+    def test_PortscanAttack_open(self):
+        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=80']], time_limit=1.5,
+                                      factor=1000)
 
-    def test_PortscanAttack_close_10_000(self):
-        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=20']], time_limit=15,
-                                      factor=10000)
+    def test_PortscanAttack_close(self):
+        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=20']], time_limit=1.5,
+                                      factor=1000)
 
-    def test_SQLi_10_000(self):
-        # FIXME: sometimes it takes 15.34028493521018 instead of the normal 7.150923313737726 seconds
-        self.temporal_efficiency_test([['SQLiAttack', 'ip.dst=192.168.0.1']], time_limit=15, factor=10000)
+    def test_SQLi(self):
+        self.temporal_efficiency_test([['SQLiAttack', 'ip.dst=192.168.0.1']], time_limit=1.5, factor=1000)
 
-    def test_Joomla_10_000(self):
-        self.temporal_efficiency_test([['JoomlaRegPrivExploit', 'ip.src=192.168.178.1']], time_limit=15, factor=10000)
+    def test_Joomla(self):
+        self.temporal_efficiency_test([['JoomlaRegPrivExploit', 'ip.src=192.168.178.1']], time_limit=1.5, factor=1000)
 
-    def test_SalityBotnet_10_000(self):
-        self.temporal_efficiency_test([['SalityBotnet']], time_limit=15, factor=10000)
+    def test_SalityBotnet(self):
+        self.temporal_efficiency_test([['SalityBotnet']], time_limit=1.5, factor=1000)
 
     @mock.patch('Attack.BaseAttack.BaseAttack.write_attack_pcap', side_effect=Lib.write_attack_pcap)
-    def test_DDoS_10_000(self, mock_write_attack_pcap):
-        # TODO: update attack args, when DDoS gets refactored
-        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=100', 'packets.per-second=95',
-                                        'attack.duration=150']], time_limit=15, factor=10000)
+    def test_DDoS(self, mock_write_attack_pcap):
+        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=10', 'packets.per-second=95',
+                                        'attack.duration=10']], time_limit=1.5, factor=1000)
 
-    @mock.patch('Attack.BaseAttack.BaseAttack.write_attack_pcap', side_effect=Lib.write_attack_pcap)
-    def test_DDoS_100_000(self, mock_write_attack_pcap):
-        # TODO: update attack args, when DDoS gets refactored
-        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=1000', 'packets.per-second=950',
-                                        'attack.duration=300']], time_limit=150, factor=100000)
+    def test_MS17(self):
+        self.temporal_efficiency_test([['MS17Scan', 'ip.src=192.168.178.1']], time_limit=1.5, factor=1000)
 
-        # TODO: add temporal efficiency test(s) for EternalBlue and MS17
+    # FIXME: improve EternalBlue efficiency
+    #def test_EternalBlue(self):
+    #    self.temporal_efficiency_test([['EternalBlue']], time_limit=1.5, factor=1000)

code/Test/test_Queries.py

@@ -235,4 +235,4 @@ class TestQueries(unittest.TestCase):
         self.assertEqual(controller.statistics.process_db_query('all(protocolname)'), ['IPv4', 'TCP', 'UDP'])
 
     def test_nested_query(self):
-        self.assertEqual(controller.statistics.process_db_query('macaddress(ipaddress=most_used(ipaddress))'), '08:00:27:a3:83:43')
+        self.assertEqual(controller.statistics.process_db_query('macaddress(ipaddress in most_used(ipaddress))'), '08:00:27:a3:83:43')