Merge branch 'temporal_efficiency_fix' of stefan.schmidt/ID2T-toolkit into master

code/Attack/DDoSAttack.py

@@ -55,7 +55,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
         # attacker configuration
         num_attackers = rnd.randint(1, 16)
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
 
         self.add_param_value(atkParam.Parameter.IP_SOURCE,
                              self.generate_random_ipv4_address(most_used_ip_class, num_attackers))
@@ -82,7 +82,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
         if (num_attackers is not None) and (num_attackers is not 0):
             # user supplied atkParam.Parameter.NUMBER_ATTACKERS
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             # Create random attackers based on user input atkParam.Parameter.NUMBER_ATTACKERS
             ip_source_list = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
             mac_source_list = self.generate_random_mac_address(num_attackers)
@@ -164,14 +164,14 @@ class DDoSAttack(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
             destination_win_value = destination_win_prob_dict.random()
         else:
-            destination_win_value = self.statistics.process_db_query("most_used(winSize)")
+            destination_win_value = self.statistics.get_most_used_win_size()
 
         destination_win_value = Util.handle_most_used_outputs(destination_win_value)
 
         # MSS that was used by IP destination in background traffic
         mss_dst = self.statistics.get_most_used_mss(ip_destination)
         if mss_dst is None:
-            mss_dst = self.statistics.process_db_query("most_used(mssValue)")
+            mss_dst = self.statistics.get_most_used_mss_value()
 
         mss_dst = Util.handle_most_used_outputs(mss_dst)
 

code/Attack/EternalBlueExploit.py

@@ -106,7 +106,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -114,7 +114,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -132,7 +132,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
 
         # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-        mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+        mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         if not mss_value:
             mss_value = 1465
 

code/Attack/FTPWinaXeExploit.py

@@ -50,7 +50,7 @@ class FTPWinaXeExploit(BaseAttack.BaseAttack):
         most_used_ip_address = self.statistics.get_most_used_ip_address()
 
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
         attacker_ip = self.generate_random_ipv4_address(most_used_ip_class)
         self.add_param_value(atkParam.Parameter.IP_DESTINATION, attacker_ip)
         self.add_param_value(atkParam.Parameter.MAC_DESTINATION, self.generate_random_mac_address())
@@ -98,7 +98,7 @@ class FTPWinaXeExploit(BaseAttack.BaseAttack):
         # Create random victim if specified
         if self.get_param_value(atkParam.Parameter.IP_SOURCE_RANDOMIZE):
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             ip_victim = self.generate_random_ipv4_address(most_used_ip_class, 1)
             mac_victim = self.generate_random_mac_address()
 

code/Attack/JoomlaRegPrivExploit.py

@@ -107,7 +107,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -115,7 +115,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Inject Joomla_registration_privesc
         # Read joomla_registration_privesc pcap file

code/Attack/MS17ScanAttack.py

@@ -104,7 +104,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -112,7 +112,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -130,7 +130,7 @@ class MS17ScanAttack(BaseAttack.BaseAttack):
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
 
         # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-        mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+        mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         if not mss_value:
             mss_value = 1465
 

code/Attack/PortscanAttack.py

@@ -139,14 +139,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_mss_prob_dict = lea.Lea.fromValFreqsDict(source_mss_dist)
             source_mss_value = source_mss_prob_dict.random()
         else:
-            source_mss_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(mssValue)"))
+            source_mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
         destination_mss_dist = self.statistics.get_mss_distribution(ip_destination)
         if len(destination_mss_dist) > 0:
             destination_mss_prob_dict = lea.Lea.fromValFreqsDict(destination_mss_dist)
             destination_mss_value = destination_mss_prob_dict.random()
         else:
-            destination_mss_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(mssValue)"))
+            destination_mss_value = Util.handle_most_used_outputs(self.statistics.get_most_used_mss_value())
 
         # Set TTL based on TTL distribution of IP address
         source_ttl_dist = self.statistics.get_ttl_distribution(ip_source)
@@ -154,14 +153,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
             destination_ttl_prob_dict = lea.Lea.fromValFreqsDict(destination_ttl_dist)
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
-            destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+            destination_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         # Set Window Size based on Window Size distribution of IP address
         source_win_dist = self.statistics.get_win_distribution(ip_source)
@@ -169,14 +167,13 @@ class PortscanAttack(BaseAttack.BaseAttack):
             source_win_prob_dict = lea.Lea.fromValFreqsDict(source_win_dist)
             source_win_value = source_win_prob_dict.random()
         else:
-            source_win_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(winSize)"))
+            source_win_value = Util.handle_most_used_outputs(self.statistics.get_most_used_win_size())
         destination_win_dist = self.statistics.get_win_distribution(ip_destination)
         if len(destination_win_dist) > 0:
             destination_win_prob_dict = lea.Lea.fromValFreqsDict(destination_win_dist)
             destination_win_value = destination_win_prob_dict.random()
         else:
-            destination_win_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(winSize)"))
+            destination_win_value = Util.handle_most_used_outputs(self.statistics.get_most_used_win_size())
 
         min_delay, max_delay = self.get_reply_delay(ip_destination)
 

code/Attack/SMBLorisAttack.py

@@ -48,7 +48,7 @@ class SMBLorisAttack(BaseAttack.BaseAttack):
         most_used_ip_address = self.statistics.get_most_used_ip_address()
 
         # The most used IP class in background traffic
-        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+        most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
         num_attackers = rnd.randint(1, 16)
         source_ip = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
 
@@ -89,7 +89,7 @@ class SMBLorisAttack(BaseAttack.BaseAttack):
         # user supplied atkParam.Parameter.NUMBER_ATTACKERS
         if (num_attackers is not None) and (num_attackers is not 0):
             # The most used IP class in background traffic
-            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ipClass)"))
+            most_used_ip_class = Util.handle_most_used_outputs(self.statistics.get_most_used_ip_class())
             # Create random attackers based on user input atkParam.Parameter.NUMBER_ATTACKERS
             ip_source = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
             mac_source = self.generate_random_mac_address(num_attackers)

code/Attack/SQLiAttack.py

@@ -111,7 +111,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
             source_ttl_prob_dict = lea.Lea.fromValFreqsDict(source_ttl_dist)
             source_ttl_value = source_ttl_prob_dict.random()
         else:
-            source_ttl_value = Util.handle_most_used_outputs(self.statistics.process_db_query("most_used(ttlValue)"))
+            source_ttl_value = Util.handle_most_used_outputs(self.statistics.get_most_used_ttl_value())
 
         destination_ttl_dist = self.statistics.get_ttl_distribution(ip_destination)
         if len(destination_ttl_dist) > 0:
@@ -119,7 +119,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
             destination_ttl_value = destination_ttl_prob_dict.random()
         else:
             destination_ttl_value = Util.handle_most_used_outputs(
-                self.statistics.process_db_query("most_used(ttlValue)"))
+                self.statistics.get_most_used_ttl_value())
 
         # Inject SQLi Attack
         # Read SQLi Attack pcap file

code/Core/Statistics.py

@@ -544,7 +544,7 @@ class Statistics:
         :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of
         randomly chosen IP addresses
         """
-        ip_address_list = self.process_db_query("all(ipAddress)")
+        ip_address_list = self.process_db_query("SELECT ipAddress from ip_statistics ORDER BY ipAddress ASC")
         if count == 1:
             return random.choice(ip_address_list)
         else:
@@ -560,13 +560,45 @@ class Statistics:
         :param mac_address: the MAC address of which the IP shall be returned, if existing in DB
         :return: the IP address used in the dataset by a given MAC address
         """
-        return self.process_db_query('ipAddress(macAddress=' + mac_address + ")")
+        return self.process_db_query("SELECT DISTINCT ipAddress FROM ip_mac WHERE macAddress = '" + mac_address + "'")
 
     def get_mac_address(self, ip_address: str):
         """
         :return: The MAC address used in the dataset for the given IP address.
         """
-        return self.process_db_query('macAddress(ipAddress=' + ip_address + ")")
+        return self.process_db_query("SELECT DISTINCT macAddress from ip_mac WHERE ipAddress = '" + ip_address + "'")
+
+    def get_most_used_ttl_value(self):
+        """
+        :return: The most used TTL value.
+        """
+        return self.process_db_query("SELECT ttlValue FROM (SELECT ttlValue, SUM(ttlCount) as occ FROM ip_ttl GROUP BY "
+                                     "ttlValue) WHERE occ=(SELECT SUM(ttlCount) as occ FROM ip_ttl GROUP BY ttlValue "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY ttlValue ASC")
+
+    def get_most_used_ip_class(self):
+        """
+        :return: The most used IP class.
+        """
+        return self.process_db_query("SELECT ipClass FROM (SELECT ipClass, COUNT(*) as occ from ip_statistics GROUP BY "
+                                     "ipClass ORDER BY occ DESC) WHERE occ=(SELECT COUNT(*) as occ from ip_statistics "
+                                     "GROUP BY ipClass ORDER BY occ DESC LIMIT 1) ORDER BY ipClass ASC")
+
+    def get_most_used_win_size(self):
+        """
+        :return: The most used window size.
+        """
+        return self.process_db_query("SELECT winSize FROM (SELECT winSize, SUM(winCount) as occ FROM tcp_win GROUP BY "
+                                     "winSize) WHERE occ=(SELECT SUM(winCount) as occ FROM tcp_win GROUP BY winSize "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY winSize ASC")
+
+    def get_most_used_mss_value(self):
+        """
+        :return: The most used mss value.
+        """
+        return self.process_db_query("SELECT mssValue FROM (SELECT mssValue, SUM(mssCount) as occ FROM tcp_mss GROUP BY"
+                                     " mssValue) WHERE occ=(SELECT SUM(mssCount) as occ FROM tcp_mss GROUP BY mssValue "
+                                     "ORDER BY occ DESC LIMIT 1) ORDER BY mssValue ASC")
 
     def get_most_used_mss(self, ip_address: str):
         """

code/Test/efficiency_testing.py

@@ -5,65 +5,50 @@ import Test.ID2TAttackTest as Test
 
 
 class EfficiencyTests(Test.ID2TAttackTest):
-    def test_SMBLoris_10_000(self):
-        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=30', 'packets.per-second=8.0']],
-                                      time_limit=15, factor=10000)
+    def test_SMBLoris(self):
+        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=4', 'packets.per-second=8.0']],
+                                      time_limit=1.5, factor=1000)
 
-    def test_SMBLoris_100_000(self):
-        self.temporal_efficiency_test([['SMBLorisAttack', 'attackers.count=30', 'packets.per-second=98']],
-                                      time_limit=150, factor=100000)
-
-    def test_SMBScan_10_000(self):
+    def test_SMBScan(self):
         self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1',
-                                        'ip.dst=192.168.178.10-192.168.197.145']], time_limit=15, factor=10000)
-
-    def test_SMBScan_100_000(self):
-        self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1', 'ip.dst=192.168.0.1-192.168.195.76']],
-                                      time_limit=150, factor=100000)
+                                        'ip.dst=192.168.178.10-192.168.179.253']], time_limit=1.5, factor=1000)
 
-    def test_SMBScan_hosting_10_000(self):
+    def test_SMBScan_hosting(self):
         self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1',
-                                        'ip.dst=192.168.178.10-192.168.181.241',
-                                        'hosting.ip=192.168.178.10-192.168.181.241']], time_limit=15, factor=10000)
-
-    def test_SMBScan_hosting_100_000(self):
-        self.temporal_efficiency_test([['SMBScanAttack', 'ip.src=192.168.178.1', 'ip.dst=192.168.178.10-192.168.217.25',
-                                        'hosting.ip=192.168.178.10-192.168.217.25']], time_limit=150, factor=100000)
+                                        'ip.dst=192.168.178.10-192.168.178.109',
+                                        'hosting.ip=192.168.178.10-192.168.178.109']], time_limit=1.5, factor=1000)
 
     @mock.patch('ID2TLib.Utility.get_rnd_bytes', side_effect=Lib.get_bytes)
     @mock.patch('ID2TLib.Utility.get_rnd_x86_nop', side_effect=Lib.get_x86_nop)
     def test_FTPExploit(self, mock_get_rnd_x86_nop, mock_get_rnd_bytes):
         self.temporal_efficiency_test([['FTPWinaXeExploit', 'ip.src=192.168.178.1', 'ip.dst=192.168.178.10']],
-                                      time_limit=15, factor=10000)
+                                      time_limit=1.5, factor=1000)
 
-    def test_PortscanAttack_open_10_000(self):
-        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=80']], time_limit=15,
-                                      factor=10000)
+    def test_PortscanAttack_open(self):
+        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=80']], time_limit=1.5,
+                                      factor=1000)
 
-    def test_PortscanAttack_close_10_000(self):
-        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=20']], time_limit=15,
-                                      factor=10000)
+    def test_PortscanAttack_close(self):
+        self.temporal_efficiency_test([['PortscanAttack', 'ip.src=192.168.178.1', 'port.open=20']], time_limit=1.5,
+                                      factor=1000)
 
-    def test_SQLi_10_000(self):
-        # FIXME: sometimes it takes 15.34028493521018 instead of the normal 7.150923313737726 seconds
-        self.temporal_efficiency_test([['SQLiAttack', 'ip.dst=192.168.0.1']], time_limit=15, factor=10000)
+    def test_SQLi(self):
+        self.temporal_efficiency_test([['SQLiAttack', 'ip.dst=192.168.0.1']], time_limit=1.5, factor=1000)
 
-    def test_Joomla_10_000(self):
-        self.temporal_efficiency_test([['JoomlaRegPrivExploit', 'ip.src=192.168.178.1']], time_limit=15, factor=10000)
+    def test_Joomla(self):
+        self.temporal_efficiency_test([['JoomlaRegPrivExploit', 'ip.src=192.168.178.1']], time_limit=1.5, factor=1000)
 
-    def test_SalityBotnet_10_000(self):
-        self.temporal_efficiency_test([['SalityBotnet']], time_limit=15, factor=10000)
+    def test_SalityBotnet(self):
+        self.temporal_efficiency_test([['SalityBotnet']], time_limit=1.5, factor=1000)
 
     @mock.patch('Attack.BaseAttack.BaseAttack.write_attack_pcap', side_effect=Lib.write_attack_pcap)
-    def test_DDoS_10_000(self, mock_write_attack_pcap):
-        # TODO: update attack args, when DDoS gets refactored
-        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=100', 'packets.per-second=95',
-                                        'attack.duration=150']], time_limit=15, factor=10000)
+    def test_DDoS(self, mock_write_attack_pcap):
+        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=10', 'packets.per-second=95',
+                                        'attack.duration=10']], time_limit=1.5, factor=1000)
 
-    @mock.patch('Attack.BaseAttack.BaseAttack.write_attack_pcap', side_effect=Lib.write_attack_pcap)
-    def test_DDoS_100_000(self, mock_write_attack_pcap):
-        # TODO: update attack args, when DDoS gets refactored
-        self.temporal_efficiency_test([['DDoSAttack', 'attackers.count=1000', 'packets.per-second=950',
-                                        'attack.duration=300']], time_limit=150, factor=100000)
+    def test_MS17(self):
+        self.temporal_efficiency_test([['MS17Scan', 'ip.src=192.168.178.1']], time_limit=1.5, factor=1000)
 
-        # TODO: add temporal efficiency test(s) for EternalBlue and MS17
+    # FIXME: improve EternalBlue efficiency
+    #def test_EternalBlue(self):
+    #    self.temporal_efficiency_test([['EternalBlue']], time_limit=1.5, factor=1000)