Merge branch 'develop' of https://git.tk.informatik.tu-darmstadt.de/leon.boeck/ID2T-toolkit-BotnetTraffic into develop

code/Attack/AttackParameters.py

@@ -37,7 +37,7 @@ class Parameter(Enum):
     PORT_DEST_ORDER_DESC = 'port.dst.order-desc'  # uses a descending port order instead of a ascending order
     IP_SOURCE_RANDOMIZE = 'ip.src.shuffle'  # randomizes the sources IP address if a list of IP addresses is given
     PORT_SOURCE_RANDOMIZE = 'port.src.shuffle'  # randomizes the source port if a list of sources ports is given
-    NAT_PRESENT = 'nat'  # if NAT is active, external computers cannot initiate a communication in MembersMgmtCommAttack
+    NAT_PRESENT = 'nat.present'  # if NAT is active, external computers cannot initiate a communication in MembersMgmtCommAttack
     # recommended type: Filepath ------------------------------------
     FILE_CSV = 'file.csv'  # filepath to CSV containing a communication pattern
     FILE_XML = 'file.xml'  # filepath to XML containing a communication pattern

code/Attack/MembersMgmtCommAttack.py

@@ -37,7 +37,7 @@ class Message():
         str_ = "{0}. at {1}: {2}-->{3}, {4}, refer:{5}".format(self.msg_id, self.time, self.src, self.dst, self.type, self.refer_msg_id)
         return str_
 
-from random import randint, randrange, choice
+from random import randint, randrange, choice, uniform
 from collections import deque
 from scipy.stats import gamma
 from lea import Lea
@@ -114,8 +114,8 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
 
         # PARAMETERS: initialize with default values
         # (values are overwritten if user specifies them)
-        #self.add_param_value(Param.INJECT_AFTER_PACKET, randint(0, int(self.statistics.get_packet_count()/4)))
-        self.add_param_value(Param.INJECT_AFTER_PACKET, 1)
+        self.add_param_value(Param.INJECT_AFTER_PACKET, randint(1, int(self.statistics.get_packet_count()/5)))
+        #self.add_param_value(Param.INJECT_AFTER_PACKET, 1)
 
         self.add_param_value(Param.PACKETS_PER_SECOND, 0)
         self.add_param_value(Param.FILE_XML, self.DEFAULT_XML_PATH)
@@ -295,15 +295,37 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             '''
             ids = sorted(bot_configs.keys())
             for pos,bot in enumerate(ids):
-                #print(type(bot_configs))
-                # Set TTL based on TTL distribution of IP address
-                bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
-                if len(bot_ttl_dist) > 0:
-                    source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
-                    bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
-                else:
-                    bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
+                bot_type = bot_configs[bot]["Type"]
+                # print(bot_type)
+                if(bot_type == "local"): # Set fix TTL for local Bots
+                    bot_configs[bot]["TTL"] = 128
+                    # Set TTL based on TTL distribution of IP address
+                else: # Set varying TTl for external Bots
+                    bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
+                    if len(bot_ttl_dist) > 0:
+                         source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
+                         bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
+                    else:
+                         bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
+
+
+        def add_delay(timestamp, minDelay, delay):
+            '''
+            Adds delay to a timestamp, with a minimum value of minDelay. But usually a value close to delay
+            :param timestamp: the timestamp that is to be increased
+            :param minDelay: the minimum value that is to add to the timestamp
+            :param delay: The general size of the delay. Statistically speaking: the expected value
+            :return: the updated timestamp
+            '''
 
+            randomdelay = Lea.fromValFreqsDict({0.15*delay: 7, 0.3*delay: 10, 0.7*delay:20,
+                                delay:33, 1.2*delay:20, 1.6*delay: 10, 1.9*delay: 7, 2.5*delay: 3, 4*delay: 1})
+            if 0.1*delay < minDelay:
+                print("Warning: minDelay probably too big when computing time_stamps")
+
+            general_offset = randomdelay.random()
+            unique_offset = uniform(-0.1*general_offset, 0.1*general_offset)
+            return timestamp + minDelay + general_offset + unique_offset
 
         # parse input CSV or XML
         filepath_xml = self.get_param_value(Param.FILE_XML)
@@ -376,7 +398,62 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             add_ids_to_config(sorted(external_ids), existing_external_ips, new_external_ips, bot_configs, idtype="external", router_mac=router_mac)
 
         #### Set realistic timestamps for messages ####
-        #### ... ####
+
+        most_used_ip_address = self.statistics.get_most_used_ip_address()
+        minDelay, maxDelay = self.get_reply_delay(most_used_ip_address)
+        next_timestamp = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
+        pcap_duration = float(self._get_capture_duration())
+        equi_timeslice = pcap_duration/len(messages)
+
+        # Dict, takes a tuple of 2 Bots as a key (IP with lower number first), returns the time when the Hello_reply came in
+        Hello_times = {}
+        # msg_IDs with already updated timestamps
+        updated_msgs = []
+
+        for req_msg in messages:
+            updated = 0
+            if(req_msg.msg_id in updated_msgs):
+                #message already updated
+                continue
+
+            if(req_msg.msg_id == -1):
+                #message has no corresponding request/response
+                req_msg.time = next_timestamp
+                next_timestamp = add_delay(next_timestamp, minDelay, equi_timeslice)
+                updated_msgs.append(req_msg.msg_id)
+                continue
+
+
+            elif req_msg.type != MessageType.SALITY_HELLO:
+                #Hello msg must have preceded, so make sure the timestamp of this msg is after the HELLO_REPLY
+                if int(req_msg.src) < int(req_msg.dst):
+                    hello_time = Hello_times[(req_msg.src, req_msg.dst)]
+                else:
+                    hello_time = Hello_times[(req_msg.dst, req_msg.src)] 
+                
+                if next_timestamp < hello_time:
+                    #use the time of the hello_reply instead of next_timestamp to update this pair of messages
+                    post_hello = add_delay(hello_time, minDelay, equi_timeslice)
+                    respns_msg = messages[req_msg.refer_msg_id]
+                    respns_msg.time = add_delay(post_hello, minDelay, equi_timeslice)
+                    req_msg.time = post_hello
+                    updated = 1
+
+            if not updated:
+                #update normally
+                respns_msg = messages[req_msg.refer_msg_id]
+                respns_msg.time = add_delay(next_timestamp, minDelay, equi_timeslice)
+                req_msg.time = next_timestamp
+                next_timestamp = add_delay(next_timestamp, minDelay, equi_timeslice)
+
+            updated_msgs.append(req_msg.msg_id)
+            updated_msgs.append(req_msg.refer_msg_id)
+
+            if req_msg.type == MessageType.SALITY_HELLO:
+                if int(req_msg.src) < int(req_msg.dst):
+                    Hello_times[(req_msg.src, req_msg.dst)] = respns_msg.time
+                else:
+                    Hello_times[(req_msg.dst, req_msg.src)] = respns_msg.time
         
         # create port configurations for the bots
         for bot in bot_configs:
@@ -390,6 +467,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         # put together the final messages including the full sender and receiver
         # configurations (i.e. IP, MAC, port, ...) for easier later use
         final_messages = []
+        messages = sorted(messages, key=lambda msg: msg.time)
         new_id = 0
         for msg in messages:
             type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]

code/CLI.py

@@ -3,6 +3,7 @@ import argparse
 import sys
 import random
 import numpy
+import hashlib
 
 from ID2TLib.Controller import Controller
 
@@ -64,7 +65,8 @@ class CLI(object):
         parser.add_argument('-t', '--extraTests', help='perform extra tests on the input pcap file, including calculating IP entropy'
                                                        'in interval-wise, TCP checksum, and checking payload availability.', action='store_true')
         parser.add_argument('--seed', help='random seed for the program, call the program with the same seed to get the same output')
-
+        parser.add_argument('-o', '--output', metavar="PCAP_FILE",
+                                 help='path to the output pcap file')
 
         # Attack arguments
         parser.add_argument('-a', '--attack', metavar="ATTACK", action='append',
@@ -92,9 +94,10 @@ class CLI(object):
     def seed_rng(self, seed):
         try: # try to convert the seed to int
             seed = int(seed)
-        except:
-            seed = hash(seed) # otherwise use the strings hash
-        
+        except: # otherwise use the strings hash
+            hashed_seed = hashlib.sha1(seed.encode()).digest()
+            seed = int.from_bytes(hashed_seed, byteorder="little") & 0xffffffff  # convert hash to 32-bit integer
+
         random.seed(seed)
         numpy.random.seed(seed)
 
@@ -147,7 +150,7 @@ class CLI(object):
         given queries.
         """
         # Create ID2T Controller
-        controller = Controller(self.args.input, self.args.extraTests)
+        controller = Controller(self.args.input, self.args.extraTests, self.args.output)
 
         # Load PCAP statistics
         controller.load_pcap_statistics(self.args.export, self.args.recalculate, self.args.statistics)

code/ID2TLib/CommunicationProcessor.py

@@ -5,7 +5,6 @@ from Attack.MembersMgmtCommAttack import Message
 # needed because of machine inprecision. E.g A time difference of 0.1s is stored as >0.1s
 EPS_TOLERANCE = 1e-13  # works for a difference of 0.1, no less
 
-######### TODO: WIKI ADD VALUE RANGES ##########
 
 class CommunicationProcessor():
     """

code/ID2TLib/Controller.py

@@ -8,13 +8,17 @@ from ID2TLib.Statistics import Statistics
 
 
 class Controller:
-    def __init__(self, pcap_file_path: str, do_extra_tests: bool):
+    def __init__(self, in_pcap_file_path: str, do_extra_tests: bool, out_pcap_file_path):
         """
         Creates a new Controller, acting as a central coordinator for the whole application.
         :param pcap_file_path:
         """
         # Fields
-        self.pcap_src_path = pcap_file_path.strip()
+        self.pcap_src_path = in_pcap_file_path.strip()
+        if out_pcap_file_path:
+            self.pcap_out_path = out_pcap_file_path.strip()
+        else:
+            self.pcap_out_path = None
         self.pcap_dest_path = ''
         self.written_pcaps = []
         self.do_extra_tests = do_extra_tests
@@ -69,6 +73,12 @@ class Controller:
         print("Merging base pcap with single attack pcap...", end=" ")
         sys.stdout.flush()  # force python to print text immediately
         self.pcap_dest_path = self.pcap_file.merge_attack(attacks_pcap_path)
+        if self.pcap_out_path:
+            if not self.pcap_out_path.endswith(".pcap"):
+                self.pcap_out_path += ".pcap"
+            os.rename(self.pcap_dest_path, self.pcap_out_path)
+            self.pcap_dest_path = self.pcap_out_path
+
         print("done.")
 
         # delete intermediate PCAP files

code/ID2TLib/GatherInformationOfIpA.py

@@ -0,0 +1,69 @@
+import subprocess
+
+# a function that gathers more information about a given IP Address
+def gatherInformationOfIpA(ipToCheck):
+    descr = []
+    country = []
+    source = []
+    autSys = []
+    nothingFound = False
+    descrFound = False
+    countryFound = False
+    sourceFound = False
+
+    # execute a shell command and save it to t
+    t = subprocess.run(['whois', ipToCheck], stdout=subprocess.PIPE)
+
+    # save generated output of shell command to a file
+    with open("output.txt", "w") as output:
+        output.write(t.stdout.decode('utf-8'))
+
+    # parse information, like Description, Country, Source and if found the ASN
+    with open("output.txt", "r", encoding="utf-8", errors='replace') as ripeDb:
+        ipInfos = [line.split() for line in ripeDb if line.strip()]
+
+        for i, row in enumerate(ipInfos):
+            if any("inetnum" in s for s in row):
+                if ipToCheck >= row[1] and ipToCheck <= row[3]:
+                    for local in range(1, 20):
+                        if ("descr:" in ipInfos[i + local]) and not descrFound:
+                            descr.extend(ipInfos[i + local][1:])
+                            descrFound = True
+                            continue
+                        if ("country:" in ipInfos[i + local]) and not countryFound:
+                            country.extend(ipInfos[i + local][1:])
+                            countryFound = True
+                            continue
+                        if ("source:" in ipInfos[i + local]) and not sourceFound:
+                            source.extend(ipInfos[i + local][1:])
+                            sourceFound = True
+                            break
+            if any("origin" in s for s in row):
+                autSys.extend(row[1:])
+                break
+        if not descrFound or not countryFound or not sourceFound:
+            nothingFound = True
+
+
+    # print information (which use of this information is wanted? Output, Returned?)
+    if not nothingFound:
+        print("#############################################")
+        print("More Information about", ipToCheck)
+        print("Description: ", ' '.join(descr) if descr else "unknown")
+        print("Country:     ", ' '.join(country) if country else "unknown")
+        print("Source:      ", ' '.join(source) if source else "unknown")
+        print("AS Number:   ", ' '.join(autSys) if autSys else "unknown")
+        print("#############################################")
+        print("\n")
+    else:
+        print("IP-Address", ipToCheck, "is not assigned by IANA yet\n")
+
+    # in case it should be stored to a file
+    with open("information.txt", "w") as info:
+        info.write("#############################################\n")
+        info.write("More Information about" + ipToCheck + "\n")
+        info.write("Description: " + ' '.join(descr) + "\n" if descr else "unknown" + "\n")
+        info.write("Country:     " + ' '.join(country) + "\n" if country else "unknown" + "\n")
+        info.write("Source:      " + ' '.join(source) + "\n" if source else "unknown" + "\n")
+        info.write("AS Number:   " + ' '.join(autSys) + "\n" if autSys else "unknown" + "\n")
+        info.write("#############################################\n")