code/Attack/AttackParameters.py

@@ -10,8 +10,6 @@ class Parameter(Enum):
     IP_SOURCE = 'ip.src'  # source IP address
     IP_DESTINATION = 'ip.dst'  # destination IP address
     IP_DNS = 'ip.dns'  # IP address of DNS server
-    HOSTING_IP = 'hosting.ip'
-    IP_DESTINATION_END = 'ip.dst.end'
     # recommended type: MAC address ------------------------------
     MAC_SOURCE = 'mac.src'  # MAC address of source
     MAC_DESTINATION = 'mac.dst'  # MAC address of destination
@@ -25,6 +23,7 @@ class Parameter(Enum):
     ATTACK_DURATION = 'attack.duration' # in seconds
     VICTIM_BUFFER = 'victim.buffer' # in packets
     TARGET_URI = 'target.uri'
+    NUMBER_INITIATOR_BOTS = 'bots.count'
     # recommended type: domain -----------------------------------
     TARGET_HOST = 'target.host'
 
@@ -38,13 +37,19 @@ class Parameter(Enum):
     PORT_DEST_ORDER_DESC = 'port.dst.order-desc'  # uses a descending port order instead of a ascending order
     IP_SOURCE_RANDOMIZE = 'ip.src.shuffle'  # randomizes the sources IP address if a list of IP addresses is given
     PORT_SOURCE_RANDOMIZE = 'port.src.shuffle'  # randomizes the source port if a list of sources ports is given
-
-    PROTOCOL_VERSION = 'protocol.version'
-    HOSTING_VERSION = 'hosting.version'
-    SOURCE_PLATFORM = 'src.platform'
-    CUSTOM_PAYLOAD = 'custom.payload'  # custom payload for ftp exploits
-    CUSTOM_PAYLOAD_FILE = 'custom.payload.file'  # file that contains custom payload for ftp exploits
-
+    NAT_PRESENT = 'nat.present'  # if NAT is active, external computers cannot initiate a communication in MembersMgmtCommAttack
+    TTL_FROM_CAIDA = 'ttl.from.caida'  # if True, TTLs are assigned based on the TTL distributions from the CAIDA dataset
+    # recommended type: Filepath ------------------------------------
+    FILE_CSV = 'file.csv'  # filepath to CSV containing a communication pattern
+    FILE_XML = 'file.xml'  # filepath to XML containing a communication pattern
+    # recommended type: CommType ------------------------------------
+    COMM_TYPE = "comm.type"  # the locality of bots in botnet communication (e.g. local, external, mixed)
+    # recommended type: Percentage (0.0-1.0) ------------------------------------
+    IP_REUSE_TOTAL = 'ip.reuse.total'  # percentage of IPs in original PCAP to be reused
+    IP_REUSE_LOCAL = 'ip.reuse.local'  # percentage of private IPs in original PCAP to be reused
+    IP_REUSE_EXTERNAL = 'ip.reuse.external'  # percentage of public IPs in original PCAP to be reused
+    # recommended type: Positive Integer between 0 and 100 ------------------------------------
+    PACKET_PADDING = 'packet.padding'
 
 class ParameterTypes(Enum):
     """
@@ -60,4 +65,7 @@ class ParameterTypes(Enum):
     TYPE_FLOAT = 6
     TYPE_PACKET_POSITION = 7  # used to derive timestamp from parameter INJECT_AFTER_PACKET
     TYPE_DOMAIN = 8
-    TYPE_STRING = 9
+    TYPE_FILEPATH = 9
+    TYPE_COMM_TYPE = 10
+    TYPE_PERCENTAGE = 11
+    TYPE_PADDING = 12

code/Attack/BaseAttack.py

@@ -5,17 +5,16 @@ import os
 import random
 import re
 import tempfile
-import numpy as np
-
 from abc import abstractmethod, ABCMeta
 from scapy.layers.inet import Ether
+import numpy as np, math
+
+import ID2TLib.libpcapreader as pr
 from scapy.utils import PcapWriter
 
 from Attack import AttackParameters
 from Attack.AttackParameters import Parameter
 from Attack.AttackParameters import ParameterTypes
-import ID2TLib.libpcapreader as pr
-
 
 class BaseAttack(metaclass=ABCMeta):
     """
@@ -241,6 +240,50 @@ class BaseAttack(metaclass=ABCMeta):
         domain = re.match('^(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$', val)
         return (domain is not None)
 
+    @staticmethod
+    def _is_filepath(val: str):
+        """
+        Verifies that the given string points to an existing file
+
+        :param filepath: The filepath as string
+        :return: True if the file at the given location exists, otherwise False
+        """
+        return os.path.isfile(val)
+
+    @staticmethod
+    def _is_comm_type(val: str):
+        """
+        Verifies that the given string is a valid communications type
+
+        :param comm_type: the type of communication as a string
+        :return: True if the given type is a valid communications type, otherwise False
+        """
+        comm_types = {"local", "external", "mixed"}
+        return val in comm_types
+
+    @staticmethod
+    def _is_percentage(val: float):
+        """
+        Verifies that the given float value is a valid percentage, i.e. between 0 and 1
+
+        :param percentage: the float to test for validity
+        :return: True if the given type is a valid percentage, otherwise False
+        """
+        if val >= 0 and val <= 1:
+            return True
+        return False
+
+    @staticmethod
+    def _is_padding(val: int):
+        """
+        Verifies that the given int is a valid padding size, i.e. between 0 and 100
+
+        :param padding: the padding to test for its size
+        :return: True if the given type is valid padding, False otherwise
+        """
+        if val >= 0 and val <= 100:
+            return True
+        return False
 
     #########################################
     # HELPER METHODS
@@ -301,9 +344,6 @@ class BaseAttack(metaclass=ABCMeta):
             elif isinstance(value, str) and value.isdigit() and int(value) >= 0:
                 is_valid = True
                 value = int(value)
-        elif param_type == ParameterTypes.TYPE_STRING:
-            if isinstance(value, str):
-                is_valid = True
         elif param_type == ParameterTypes.TYPE_FLOAT:
             is_valid, value = self._is_float(value)
             # this is required to avoid that the timestamp's microseconds of the first attack packet is '000000'
@@ -312,6 +352,15 @@ class BaseAttack(metaclass=ABCMeta):
             # e.g. inject.at-timestamp=123456 -> is changed to: 123456.[random digits]
             if param_name == Parameter.INJECT_AT_TIMESTAMP and is_valid and ((value - int(value)) == 0):
                 value = value + random.uniform(0, 0.999999)
+            # first packet of a pcap displays a timestamp of zero, but internally (usually) has a much larger one
+            # inject.at-timestamp has to be shifted by the value of the first packet of the input pcap
+            # otherwise new packets are always injected at the beginning and there is a large distance
+            # to the packets of the input pcap
+            if param_name == Parameter.INJECT_AT_TIMESTAMP and is_valid:
+                ts_first_pkt = pr.pcap_processor(self.statistics.pcap_filepath, "False").get_timestamp_mu_sec(1)
+                if ts_first_pkt >= 0:
+                    is_valid = True
+                    value = value + (ts_first_pkt / 1000000)  # convert microseconds from getTimestampMuSec into seconds
         elif param_type == ParameterTypes.TYPE_TIMESTAMP:
             is_valid = self._is_timestamp(value)
         elif param_type == ParameterTypes.TYPE_BOOLEAN:
@@ -324,6 +373,25 @@ class BaseAttack(metaclass=ABCMeta):
                 value = (ts / 1000000)  # convert microseconds from getTimestampMuSec into seconds
         elif param_type == ParameterTypes.TYPE_DOMAIN:
             is_valid = self._is_domain(value)
+        elif param_type == ParameterTypes.TYPE_FILEPATH:
+            is_valid = self._is_filepath(value)
+        elif param_type == ParameterTypes.TYPE_COMM_TYPE:
+            is_valid = self._is_comm_type(value)
+        elif param_type == ParameterTypes.TYPE_PERCENTAGE:
+            is_valid, value = self._is_float(value)
+            if is_valid and (param_name in {Parameter.IP_REUSE_TOTAL, Parameter.IP_REUSE_LOCAL, Parameter.IP_REUSE_EXTERNAL}):
+                is_valid = self._is_percentage(value)
+            else: 
+                is_valid = False
+        elif param_type == ParameterTypes.TYPE_PADDING:
+            if isinstance(value, int):
+                is_valid = True
+            elif isinstance(value, str) and value.isdigit():
+                is_valid = True
+                value = int(value)
+                
+            if is_valid:
+                is_valid = self._is_padding(value) 
 
         # add value iff validation was successful
         if is_valid:
@@ -384,10 +452,17 @@ class BaseAttack(metaclass=ABCMeta):
 
         return destination
 
-    def get_reply_delay(self, ip_dst):
+    def post_pcap_written(self, final_filename):
+        """
+        :param final_filename: The filename of the final pcap created
+        """
+        pass
+
+    def get_reply_delay(self, ip_dst, default = 2000):
         """
            Gets the minimum and the maximum reply delay for all the connections of a specific IP.
            :param ip_dst: The IP to reterive its reply delay.
+           :param default: The default value to return if no delay could be fount. If < 0 raise an exception instead
            :return minDelay: minimum delay
            :return maxDelay: maximum delay
 
@@ -402,6 +477,14 @@ class BaseAttack(metaclass=ABCMeta):
             minDelay = np.median(allMinDelays)
             allMaxDelays = self.statistics.process_db_query("SELECT maxDelay FROM conv_statistics LIMIT 500;")
             maxDelay = np.median(allMaxDelays)
+
+            if math.isnan(minDelay): # maxDelay is nan too then
+                if default < 0:
+                    raise ValueError("Could not calculate min/maxDelay")
+
+                minDelay = default
+                maxDelay = default
+
         minDelay = int(minDelay) * 10 ** -6  # convert from micro to seconds
         maxDelay = int(maxDelay) * 10 ** -6
         return minDelay, maxDelay
@@ -490,7 +573,7 @@ class BaseAttack(metaclass=ABCMeta):
         inter_arrival_times = []
         prvsPktTime = 0
         for index, pkt in enumerate(packets):
-            timestamp = pkt[2][0] + pkt[2][1]/10**6
+            timestamp = pkt[1][0] + pkt[1][1]/10**6
 
             if index == 0:
                 prvsPktTime = timestamp

code/Attack/DDoSAttack.py

@@ -1,18 +1,17 @@
 import logging
+from random import randint, uniform, choice
 
-from random import randint, choice
 from lea import Lea
-from collections import deque
 from scipy.stats import gamma
-from scapy.layers.inet import IP, Ether, TCP, RandShort
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
 from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, get_interval_pps, get_nth_random_element, index_increment
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
+from scapy.layers.inet import IP, Ether, TCP, RandShort
+from collections import deque
 
 
 class DDoSAttack(BaseAttack.BaseAttack):
@@ -72,7 +71,55 @@ class DDoSAttack(BaseAttack.BaseAttack):
         self.add_param_value(Param.MAC_DESTINATION, destination_mac)
         self.add_param_value(Param.VICTIM_BUFFER, randint(1000,10000))
 
-    def generate_attack_pcap(self):
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps, delay=0):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            if delay == 0:
+                # Calculate the request timestamp
+                # A distribution to imitate the bursty behavior of traffic
+                randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+                return timestamp + uniform(1 / pps, randomdelay.random())
+            else:
+                # Calculate the reply timestamp
+                randomdelay = Lea.fromValFreqsDict({2 * delay: 70, 3 * delay: 20, 5 * delay: 7, 10 * delay: 3})
+                return timestamp + uniform(1 / pps + delay, 1 / pps + randomdelay.random())
+
+        def get_nth_random_element(*element_list):
+            """
+            Returns the n-th element of every list from an arbitrary number of given lists.
+            For example, list1 contains IP addresses, list 2 contains MAC addresses. Use of this function ensures that
+            the n-th IP address uses always the n-th MAC address.
+            :param element_list: An arbitrary number of lists.
+            :return: A tuple of the n-th element of every list.
+            """
+            range_max = min([len(x) for x in element_list])
+            if range_max > 0: range_max -= 1
+            n = randint(0, range_max)
+            return tuple(x[n] for x in element_list)
+
+        def index_increment(number: int, max: int):
+            if number + 1 < max:
+                return number + 1
+            else:
+                return 0
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) for a specific time interval.
+            :param complement_interval_pps: an array of tuples (the last timestamp in the interval, the packet rate in the crresponding interval).
+            :param timestamp: the timestamp at which the packet rate is required.
+            :return: the corresponding packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp <= row[0]:
+                    return row[1]
+            # In case the timestamp > capture max timestamp
+            return complement_interval_pps[-1][1]
+
         def get_attacker_config(ipAddress: str):
             """
             Returns the attacker configuration depending on the IP address, this includes the port for the next
@@ -104,6 +151,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
                 attacker_ttl_mapping[ipAddress] = ttl
             # return port and TTL
             return next_port, ttl
+
         BUFFER_SIZE = 1000
 
         # Determine source IP and MAC address
@@ -232,7 +280,7 @@ class DDoSAttack(BaseAttack.BaseAttack):
                     replies_count+=1
                     total_pkt_num += 1
 
-                attacker_pps = max(get_interval_pps(complement_interval_attacker_pps, timestamp_next_pkt), (pps / num_attackers) / 2)
+                attacker_pps = max(getIntervalPPS(complement_interval_attacker_pps, timestamp_next_pkt), (pps/num_attackers)/2)
                 timestamp_next_pkt = update_timestamp(timestamp_next_pkt, attacker_pps)
 
                 # Store timestamp of first packet (for attack label)

code/Attack/EternalBlueExploit.py

@@ -1,23 +1,22 @@
 import logging
-
 from random import randint, uniform
+
 from lea import Lea
-from scapy.utils import RawPcapReader
-from scapy.layers.inet import Ether
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
 from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, get_interval_pps
-from ID2TLib.SMBLib import smb_port
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
-
+from scapy.utils import RawPcapReader
+from scapy.layers.inet import IP, Ether, TCP, RandShort
 
 class EternalBlueExploit(BaseAttack.BaseAttack):
     template_scan_pcap_path = "resources/Win7_eternalblue_scan.pcap"
     template_attack_pcap_path = "resources/Win7_eternalblue_exploit.pcap"
+    # SMB port
+    smb_port = 445
     # Empirical values from Metasploit experiments
     minDefaultPort = 30000
     maxDefaultPort = 50000
@@ -70,7 +69,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
         if isinstance(destination_mac, list) and len(destination_mac) == 0:
             destination_mac = self.generate_random_mac_address()
         self.add_param_value(Param.MAC_DESTINATION, destination_mac)
-        self.add_param_value(Param.PORT_DESTINATION, smb_port)
+        self.add_param_value(Param.PORT_DESTINATION, self.smb_port)
 
         # Attack configuration
         self.add_param_value(Param.PACKETS_PER_SECOND,
@@ -78,8 +77,28 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
                               self.statistics.get_pps_received(most_used_ip_address)) / 2)
         self.add_param_value(Param.INJECT_AFTER_PACKET, randint(0, self.statistics.get_packet_count()))
 
-    def generate_attack_pcap(self):
-
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            # Calculate the request timestamp
+            # A distribution to imitate the bursty behavior of traffic
+            randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+            return timestamp + uniform(1 / pps, randomdelay.random())
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) in specific time interval.
+
+            :return: the corresponding packet rate for packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp<=row[0]:
+                    return row[1]
+            return complement_interval_pps[-1][1] # in case the timstamp > capture max timestamp
 
         # Timestamp
         timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
@@ -152,7 +171,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
             tcp_pkt = ip_pkt.payload
 
             if pkt_num == 0:
-                if tcp_pkt.getfieldval("dport") == smb_port:
+                if tcp_pkt.getfieldval("dport") == self.smb_port:
                     orig_ip_dst = ip_pkt.getfieldval("dst") # victim IP
 
             # Request
@@ -183,7 +202,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
                 new_pkt = (eth_frame / ip_pkt / tcp_pkt)
                 new_pkt.time = timestamp_next_pkt
 
-                pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                 timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + inter_arrival_times[pkt_num]#float(timeSteps.random())
             # Reply
             else:
@@ -244,7 +263,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
                     tcp_pkt = ip_pkt.payload
 
                     if pkt_num == 0:
-                        if tcp_pkt.getfieldval("dport") == smb_port:
+                        if tcp_pkt.getfieldval("dport") == self.smb_port:
                             orig_ip_dst = ip_pkt.getfieldval("dst")
 
                     # Request
@@ -275,7 +294,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
                         new_pkt = (eth_frame / ip_pkt / tcp_pkt)
                         new_pkt.time = timestamp_next_pkt
 
-                        pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                        pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                         timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + inter_arrival_times[pkt_num] #float(timeSteps.random())
 
                     # Reply
@@ -305,7 +324,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
 
                         new_pkt = (eth_frame / ip_pkt / tcp_pkt)
 
-                        pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                        pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                         timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + inter_arrival_times[pkt_num]#float(timeSteps.random())
 
                         new_pkt.time = timestamp_next_pkt
@@ -348,7 +367,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
                         new_pkt = (eth_frame / ip_pkt / tcp_pkt)
                         new_pkt.time = timestamp_next_pkt
 
-                        pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                        pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                         timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + inter_arrival_times[pkt_num]# float(timeSteps.random())
 
                     # Reply
@@ -378,7 +397,7 @@ class EternalBlueExploit(BaseAttack.BaseAttack):
 
                         new_pkt = (eth_frame / ip_pkt / tcp_pkt)
 
-                        pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                        pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                         timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + inter_arrival_times[pkt_num]# float(timeSteps.random())
 
                         new_pkt.time = timestamp_next_pkt

code/Attack/FTPWinaXeExploit.py

@@ -1,259 +0,0 @@
-import logging
-
-from random import randint
-from lea import Lea
-from scapy.layers.inet import IP, Ether, TCP
-
-from Attack import BaseAttack
-from Attack.AttackParameters import Parameter as Param
-from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, generate_source_port_from_platform, get_rnd_x86_nop, forbidden_chars,\
-    get_rnd_bytes, get_bytes_from_file
-
-logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
-# noinspection PyPep8
-
-ftp_port = 21
-
-
-class FTPWinaXeExploit(BaseAttack.BaseAttack):
-
-    def __init__(self):
-        """
-        Creates a new instance of the FTPWinaXeExploit.
-
-        """
-        # Initialize attack
-        super(FTPWinaXeExploit, self).__init__("FTPWinaXe Exploit", "Injects an WinaXe 7.7 FTP Exploit.",
-                                               "Privilege elevation")
-
-        # Define allowed parameters and their type
-        self.supported_params = {
-            Param.IP_SOURCE: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.IP_DESTINATION: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.MAC_SOURCE: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.MAC_DESTINATION: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
-            Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
-            Param.IP_SOURCE_RANDOMIZE: ParameterTypes.TYPE_BOOLEAN,
-            Param.PACKETS_PER_SECOND: ParameterTypes.TYPE_FLOAT,
-            Param.CUSTOM_PAYLOAD: ParameterTypes.TYPE_STRING,
-            Param.CUSTOM_PAYLOAD_FILE: ParameterTypes.TYPE_STRING
-        }
-
-    def init_params(self):
-        """
-        Initialize the parameters of this attack using the user supplied command line parameters.
-        Use the provided statistics to calculate default parameters and to process user
-        supplied queries.
-
-        """
-        # PARAMETERS: initialize with default values
-        # (values are overwritten if user specifies them)
-        most_used_ip_address = self.statistics.get_most_used_ip_address()
-        if isinstance(most_used_ip_address, list):
-            most_used_ip_address = most_used_ip_address[0]
-
-        # The most used IP class in background traffic
-        most_used_ip_class = self.statistics.process_db_query("most_used(ipClass)")
-        attacker_ip = self.generate_random_ipv4_address(most_used_ip_class)
-        self.add_param_value(Param.IP_DESTINATION, attacker_ip)
-        self.add_param_value(Param.MAC_DESTINATION, self.generate_random_mac_address())
-
-        random_ip_address = self.statistics.get_random_ip_address()
-        # victim should be valid and not equal to attacker
-        while not self.is_valid_ip_address(random_ip_address) or random_ip_address == attacker_ip:
-            random_ip_address = self.statistics.get_random_ip_address()
-
-        self.add_param_value(Param.IP_SOURCE, random_ip_address)
-        victim_mac = self.statistics.get_mac_address(random_ip_address)
-        if isinstance(victim_mac, list) and len(victim_mac) == 0:
-            victim_mac = self.generate_random_mac_address()
-        self.add_param_value(Param.MAC_SOURCE, victim_mac)
-        self.add_param_value(Param.PACKETS_PER_SECOND,
-                             (self.statistics.get_pps_sent(most_used_ip_address) +
-                              self.statistics.get_pps_received(most_used_ip_address)) / 2)
-        self.add_param_value(Param.INJECT_AFTER_PACKET, randint(0, self.statistics.get_packet_count()))
-        self.add_param_value(Param.IP_SOURCE_RANDOMIZE, 'False')
-        self.add_param_value(Param.CUSTOM_PAYLOAD, '')
-        self.add_param_value(Param.CUSTOM_PAYLOAD_FILE, '')
-
-    def generate_attack_pcap(self):
-        def get_ip_data(ip_address: str):
-            """
-            :param ip_address: The ip of which (packet-)data shall be returned
-            :return: MSS, TTL and window size values of the given IP
-            """
-            # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-            mss_dist = self.statistics.get_mss_distribution(ip_address)
-            if len(mss_dist) > 0:
-                mss_prob_dict = Lea.fromValFreqsDict(mss_dist)
-                mss_value = mss_prob_dict.random()
-            else:
-                mss_value = self.statistics.process_db_query("most_used(mssValue)")
-
-            # Set TTL based on TTL distribution of IP address
-            ttl_dist = self.statistics.get_ttl_distribution(ip_address)
-            if len(ttl_dist) > 0:
-                ttl_prob_dict = Lea.fromValFreqsDict(ttl_dist)
-                ttl_value = ttl_prob_dict.random()
-            else:
-                ttl_value = self.statistics.process_db_query("most_used(ttlValue)")
-
-            # Set Window Size based on Window Size distribution of IP address
-            win_dist = self.statistics.get_win_distribution(ip_address)
-            if len(win_dist) > 0:
-                win_prob_dict = Lea.fromValFreqsDict(win_dist)
-                win_value = win_prob_dict.random()
-            else:
-                win_value = self.statistics.process_db_query("most_used(winSize)")
-
-            return mss_value, ttl_value, win_value
-
-        def check_payload_len(payload_len: int, limit: int):
-            """
-            Checks if the len of the payload exceeds a given limit
-            :param payload_len: The length of the payload
-            :param limit: The limit of the length of the payload which is allowed
-            """
-
-            if payload_len > limit:
-                print("\nCustom payload too long: ", payload_len, " bytes. Should be a maximum of ", limit, " bytes.")
-                exit(1)
-
-
-        pps = self.get_param_value(Param.PACKETS_PER_SECOND)
-
-        # Timestamp
-        timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
-        # store start time of attack
-        self.attack_start_utime = timestamp_next_pkt
-
-        # Initialize parameters
-        ip_victim = self.get_param_value(Param.IP_SOURCE)
-        ip_attacker = self.get_param_value(Param.IP_DESTINATION)
-        mac_victim = self.get_param_value(Param.MAC_SOURCE)
-        mac_attacker = self.get_param_value(Param.MAC_DESTINATION)
-
-        custom_payload = self.get_param_value(Param.CUSTOM_PAYLOAD)
-        custom_payload_len = len(custom_payload)
-        custom_payload_limit = 1000
-        check_payload_len(custom_payload_len, custom_payload_limit)
-
-        packets = []
-
-        # Create random victim if specified
-        if self.get_param_value(Param.IP_SOURCE_RANDOMIZE):
-            # The most used IP class in background traffic
-            most_used_ip_class = self.statistics.process_db_query("most_used(ipClass)")
-            ip_victim = self.generate_random_ipv4_address(most_used_ip_class, 1)
-            mac_victim = self.generate_random_mac_address()
-
-        # Get MSS, TTL and Window size value for victim/attacker IP
-        victim_mss_value, victim_ttl_value, victim_win_value = get_ip_data(ip_victim)
-        attacker_mss_value, attacker_ttl_value, attacker_win_value = get_ip_data(ip_attacker)
-
-        minDelay, maxDelay = self.get_reply_delay(ip_attacker)
-
-        attacker_seq = randint(1000, 50000)
-        victim_seq = randint(1000, 50000)
-
-        sport = generate_source_port_from_platform("win7")
-
-        # connection request from victim (client)
-        victim_ether = Ether(src=mac_victim, dst=mac_attacker)
-        victim_ip = IP(src=ip_victim, dst=ip_attacker, ttl=victim_ttl_value, flags='DF')
-        request_tcp = TCP(sport=sport, dport=ftp_port, window=victim_win_value, flags='S',
-                          seq=victim_seq, options=[('MSS', victim_mss_value)])
-        victim_seq += 1
-        syn = (victim_ether / victim_ip / request_tcp)
-        syn.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps, minDelay)
-        packets.append(syn)
-
-        # response from attacker (server)
-        attacker_ether = Ether(src=mac_attacker, dst=mac_victim)
-        attacker_ip = IP(src=ip_attacker, dst=ip_victim, ttl=attacker_ttl_value, flags='DF')
-        reply_tcp = TCP(sport=ftp_port, dport=sport, seq=attacker_seq, ack=victim_seq, flags='SA',
-                        window=attacker_win_value, options=[('MSS', attacker_mss_value)])
-        attacker_seq += 1
-        synack = (attacker_ether / attacker_ip / reply_tcp)
-        synack.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps, minDelay)
-        packets.append(synack)
-
-        # acknowledgement from victim (client)
-        ack_tcp = TCP(sport=sport, dport=ftp_port, seq=victim_seq, ack=attacker_seq, flags='A',
-                      window=victim_win_value, options=[('MSS', victim_mss_value)])
-        ack = (victim_ether / victim_ip / ack_tcp)
-        ack.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
-        packets.append(ack)
-
-        # FTP exploit packet
-        ftp_tcp = TCP(sport=ftp_port, dport=sport, seq=attacker_seq, ack=victim_seq, flags='PA',
-                        window=attacker_win_value, options=[('MSS', attacker_mss_value)])
-
-        characters = b'220'
-        characters += get_rnd_bytes(2065, forbidden_chars)
-        characters += b'\x96\x72\x01\x68'
-        characters += get_rnd_x86_nop(10, False, forbidden_chars)
-
-        custom_payload_file = self.get_param_value(Param.CUSTOM_PAYLOAD_FILE)
-
-        if custom_payload == '':
-            if custom_payload_file == '':
-                payload = get_rnd_bytes(custom_payload_limit, forbidden_chars)
-            else:
-                payload = get_bytes_from_file(custom_payload_file)
-                check_payload_len(len(payload), custom_payload_limit)
-                payload += get_rnd_x86_nop(custom_payload_limit - len(payload), False, forbidden_chars)
-        else:
-            encoded_payload = custom_payload.encode()
-            payload = get_rnd_x86_nop(custom_payload_limit - custom_payload_len, False, forbidden_chars)
-            payload += encoded_payload
-
-        characters += payload
-        characters += get_rnd_x86_nop(20, False, forbidden_chars)
-        characters += b'\r\n'
-
-        ftp_tcp.add_payload(characters)
-
-        ftp_buff = (attacker_ether / attacker_ip / ftp_tcp)
-        ftp_buff.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
-        packets.append(ftp_buff)
-        attacker_seq += len(ftp_tcp.payload)
-
-        # Fin Ack from attacker
-        fin_ack_tcp = TCP(sport=ftp_port, dport=sport, seq=attacker_seq, ack=victim_seq, flags='FA',
-                        window=attacker_win_value, options=[('MSS', attacker_mss_value)])
-
-        fin_ack = (attacker_ether / attacker_ip / fin_ack_tcp)
-        fin_ack.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps, minDelay)
-        packets.append(fin_ack)
-
-        # Ack from victim on FTP packet
-        ftp_ack_tcp = TCP(sport=sport, dport=ftp_port, seq=victim_seq, ack=attacker_seq, flags='A',
-                      window=victim_win_value, options=[('MSS', victim_mss_value)])
-        ftp_ack = (victim_ether / victim_ip / ftp_ack_tcp)
-        ftp_ack.time = timestamp_next_pkt
-        timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
-        packets.append(ftp_ack)
-
-        # Ack from victim on Fin/Ack of attacker
-        fin_ack_ack_tcp = TCP(sport=sport, dport=ftp_port, seq=victim_seq, ack=attacker_seq+1, flags='A',
-                      window=victim_win_value, options=[('MSS', victim_mss_value)])
-        fin_ack_ack = (victim_ether / victim_ip / fin_ack_ack_tcp)
-        fin_ack_ack.time = timestamp_next_pkt
-        packets.append(fin_ack_ack)
-
-        # store end time of attack
-        self.attack_end_utime = packets[-1].time
-
-        # write attack packets to pcap
-        pcap_path = self.write_attack_pcap(sorted(packets, key=lambda pkt: pkt.time))
-
-        # return packets sorted by packet time_sec_start
-        return len(packets), pcap_path

code/Attack/JoomlaRegPrivExploit.py

@@ -1,17 +1,16 @@
 import logging
+from random import randint, uniform
 
-from random import randint
 from lea import Lea
-from scapy.utils import RawPcapReader
-from scapy.layers.inet import Ether
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
 from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, get_interval_pps
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
+from scapy.utils import RawPcapReader
+from scapy.layers.inet import IP, Ether, TCP, RandShort
 
 
 class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
@@ -79,7 +78,28 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
                              (self.statistics.get_pps_sent(most_used_ip_address) +
                               self.statistics.get_pps_received(most_used_ip_address)) / 2)
 
-    def generate_attack_pcap(self):
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            # Calculate the request timestamp
+            # A distribution to imitate the bursty behavior of traffic
+            randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+            return timestamp + uniform(1 / pps, randomdelay.random())
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) in specific time interval.
+
+            :return: the corresponding packet rate for packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp <= row[0]:
+                    return row[1]
+            return complement_interval_pps[-1][1]  # in case the timstamp > capture max timestamp
 
         # Timestamp
         timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
@@ -186,7 +206,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
                 new_pkt = (eth_frame / ip_pkt/ tcp_pkt / str_tcp_seg)
                 new_pkt.time = timestamp_next_pkt
 
-                pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                 timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + float(timeSteps.random())
 
             # Reply: Victim --> attacker
@@ -212,7 +232,7 @@ class JoomlaRegPrivExploit(BaseAttack.BaseAttack):
                     victim_seq += max(strLen, 1)
 
                 new_pkt = (eth_frame / ip_pkt / tcp_pkt / str_tcp_seg)
-                pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                 timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + float(timeSteps.random())
                 new_pkt.time = timestamp_next_pkt
 

code/Attack/PortscanAttack.py

@@ -1,18 +1,17 @@
 import logging
 import csv
 
-from random import shuffle, randint, choice
+from random import shuffle, randint, choice, uniform
+
 from lea import Lea
-from scapy.layers.inet import IP, Ether, TCP
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
 from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, get_interval_pps
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
-
+from scapy.layers.inet import IP, Ether, TCP
 
 class PortscanAttack(BaseAttack.BaseAttack):
 
@@ -99,8 +98,8 @@ class PortscanAttack(BaseAttack.BaseAttack):
         if (ports_num == 1000):  # used for port.dst
             temp_array = [[0 for i in range(10)] for i in range(100)]
             port_dst_shuffled = []
-            for count in range(0, 10):
-                temp_array[count] = ports_dst[count * 100:(count + 1) * 100]
+            for count in range(0, 9):
+                temp_array[count] = ports_dst[count * 100:count * 100 + 99]
                 shuffle(temp_array[count])
                 port_dst_shuffled += temp_array[count]
         else:  # used for port.open
@@ -108,10 +107,35 @@ class PortscanAttack(BaseAttack.BaseAttack):
             port_dst_shuffled = ports_dst
         return port_dst_shuffled
 
-    def generate_attack_pcap(self):
-
-
-
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps, delay=0):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            if delay == 0:
+                # Calculate request timestamp
+                # To imitate the bursty behavior of traffic
+                randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+                return timestamp + uniform(1/pps ,  randomdelay.random())
+            else:
+                # Calculate reply timestamp
+                randomdelay = Lea.fromValFreqsDict({2*delay: 70, 3*delay: 20, 5*delay: 7, 10*delay: 3})
+                return timestamp + uniform(1 / pps + delay,  1 / pps + randomdelay.random())
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) for a specific time interval.
+
+            :param complement_interval_pps: an array of tuples (the last timestamp in the interval, the packet rate in the crresponding interval).
+            :param timestamp: the timestamp at which the packet rate is required.
+            :return: the corresponding packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp<=row[0]:
+                    return row[1]
+            return complement_interval_pps[-1][1] # in case the timstamp > capture max timestamp
 
         mac_source = self.get_param_value(Param.MAC_SOURCE)
         mac_destination = self.get_param_value(Param.MAC_DESTINATION)
@@ -255,7 +279,7 @@ class PortscanAttack(BaseAttack.BaseAttack):
 
                 # else: destination port is NOT OPEN -> no reply is sent by target
 
-            pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+            pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt),10)
             timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
 
         # store end time of attack

code/Attack/SMBLorisAttack.py

@@ -1,244 +0,0 @@
-import logging
-
-from random import randint, uniform
-from lea import Lea
-from scapy.layers.inet import IP, Ether, TCP
-from scapy.layers.netbios import NBTSession
-
-from Attack import BaseAttack
-from Attack.AttackParameters import Parameter as Param
-from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp
-from ID2TLib.SMBLib import smb_port
-
-logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
-# noinspection PyPep8
-
-
-class SMBLorisAttack(BaseAttack.BaseAttack):
-
-    def __init__(self):
-        """
-        Creates a new instance of the SMBLorisAttack.
-
-        """
-        # Initialize attack
-        super(SMBLorisAttack, self).__init__("SMBLoris Attack", "Injects an SMBLoris (D)DoS Attack",
-                                             "Resource Exhaustion")
-
-        # Define allowed parameters and their type
-        self.supported_params = {
-            Param.IP_SOURCE: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.IP_DESTINATION: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.MAC_SOURCE: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.MAC_DESTINATION: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
-            Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
-            Param.PACKETS_PER_SECOND: ParameterTypes.TYPE_FLOAT,
-            Param.ATTACK_DURATION: ParameterTypes.TYPE_INTEGER_POSITIVE,
-            Param.NUMBER_ATTACKERS: ParameterTypes.TYPE_INTEGER_POSITIVE
-        }
-
-    def init_params(self):
-        """
-        Initialize the parameters of this attack using the user supplied command line parameters.
-        Use the provided statistics to calculate default parameters and to process user
-        supplied queries.
-
-        :param statistics: Reference to a statistics object.
-        """
-        # PARAMETERS: initialize with default values
-        # (values are overwritten if user specifies them)
-        most_used_ip_address = self.statistics.get_most_used_ip_address()
-        if isinstance(most_used_ip_address, list):
-            most_used_ip_address = most_used_ip_address[0]
-
-        # The most used IP class in background traffic
-        most_used_ip_class = self.statistics.process_db_query("most_used(ipClass)")
-        num_attackers = randint(1, 16)
-        source_ip = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
-
-        self.add_param_value(Param.IP_SOURCE, source_ip)
-        self.add_param_value(Param.MAC_SOURCE, self.generate_random_mac_address(num_attackers))
-
-        random_ip_address = self.statistics.get_random_ip_address()
-        # ip-dst should be valid and not equal to ip.src
-        while not self.is_valid_ip_address(random_ip_address) or random_ip_address == source_ip:
-            random_ip_address = self.statistics.get_random_ip_address()
-
-        self.add_param_value(Param.IP_DESTINATION, random_ip_address)
-        destination_mac = self.statistics.get_mac_address(random_ip_address)
-        if isinstance(destination_mac, list) and len(destination_mac) == 0:
-            destination_mac = self.generate_random_mac_address()
-        self.add_param_value(Param.MAC_DESTINATION, destination_mac)
-        self.add_param_value(Param.PACKETS_PER_SECOND,
-                             (self.statistics.get_pps_sent(most_used_ip_address) +
-                              self.statistics.get_pps_received(most_used_ip_address)) / 2)
-        self.add_param_value(Param.INJECT_AFTER_PACKET, randint(0, self.statistics.get_packet_count()))
-        self.add_param_value(Param.ATTACK_DURATION, 30)
-
-    def generate_attack_pcap(self):
-        def get_ip_data(ip_address: str):
-            """
-            :param ip_address: the ip of which (packet-)data shall be returned
-            :return: MSS, TTL and Window Size values of the given IP
-            """
-            # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-            mss_dist = self.statistics.get_mss_distribution(ip_address)
-            if len(mss_dist) > 0:
-                mss_prob_dict = Lea.fromValFreqsDict(mss_dist)
-                mss_value = mss_prob_dict.random()
-            else:
-                mss_value = self.statistics.process_db_query("most_used(mssValue)")
-
-            # Set TTL based on TTL distribution of IP address
-            ttl_dist = self.statistics.get_ttl_distribution(ip_address)
-            if len(ttl_dist) > 0:
-                ttl_prob_dict = Lea.fromValFreqsDict(ttl_dist)
-                ttl_value = ttl_prob_dict.random()
-            else:
-                ttl_value = self.statistics.process_db_query("most_used(ttlValue)")
-
-            # Set Window Size based on Window Size distribution of IP address
-            win_dist = self.statistics.get_win_distribution(ip_address)
-            if len(win_dist) > 0:
-                win_prob_dict = Lea.fromValFreqsDict(win_dist)
-                win_value = win_prob_dict.random()
-            else:
-                win_value = self.statistics.process_db_query("most_used(winSize)")
-
-            return mss_value, ttl_value, win_value
-
-        pps = self.get_param_value(Param.PACKETS_PER_SECOND)
-
-        # Timestamp
-        first_timestamp = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
-        # store start time of attack
-        self.attack_start_utime = first_timestamp
-
-        # Initialize parameters
-        packets = []
-        ip_destination = self.get_param_value(Param.IP_DESTINATION)
-        mac_destination = self.get_param_value(Param.MAC_DESTINATION)
-
-        # Determine source IP and MAC address
-        num_attackers = self.get_param_value(Param.NUMBER_ATTACKERS)
-        if (num_attackers is not None) and (num_attackers is not 0):  # user supplied Param.NUMBER_ATTACKERS
-            # The most used IP class in background traffic
-            most_used_ip_class = self.statistics.process_db_query("most_used(ipClass)")
-            # Create random attackers based on user input Param.NUMBER_ATTACKERS
-            ip_source = self.generate_random_ipv4_address(most_used_ip_class, num_attackers)
-            mac_source = self.generate_random_mac_address(num_attackers)
-        else:  # user did not supply Param.NUMBER_ATTACKS
-            # use default values for IP_SOURCE/MAC_SOURCE or overwritten values
-            # if user supplied any values for those params
-            ip_source = self.get_param_value(Param.IP_SOURCE)
-            mac_source = self.get_param_value(Param.MAC_SOURCE)
-
-        ip_source_list = []
-        mac_source_list = []
-
-        if isinstance(ip_source, list):
-            ip_source_list = ip_source
-        else:
-            ip_source_list.append(ip_source)
-
-        if isinstance(mac_source, list):
-            mac_source_list = mac_source
-        else:
-            mac_source_list.append(mac_source)
-
-        if (num_attackers is None) or (num_attackers is 0):
-            num_attackers = min(len(ip_source_list), len(mac_source_list))
-
-        # Check ip.src == ip.dst
-        self.ip_src_dst_equal_check(ip_source_list, ip_destination)
-
-        # Get MSS, TTL and Window size value for destination IP
-        destination_mss_value, destination_ttl_value, destination_win_value = get_ip_data(ip_destination)
-
-        minDelay,maxDelay = self.get_reply_delay(ip_destination)
-
-        attack_duration = self.get_param_value(Param.ATTACK_DURATION)
-        attack_ends_time = first_timestamp + attack_duration
-
-        victim_pps = pps*num_attackers
-
-        for attacker in range(num_attackers):
-            # Get MSS, TTL and Window size value for source IP(attacker)
-            source_mss_value, source_ttl_value, source_win_value = get_ip_data(ip_source_list[attacker])
-
-            attacker_seq = randint(1000, 50000)
-            victim_seq = randint(1000, 50000)
-
-            sport = 1025
-
-            # Timestamps of first packets shouldn't be exactly the same to look more realistic
-            timestamp_next_pkt = uniform(first_timestamp, update_timestamp(first_timestamp, pps))
-
-            while timestamp_next_pkt <= attack_ends_time:
-                # Establish TCP connection
-                if sport > 65535:
-                    sport = 1025
-
-                # prepare reusable Ethernet- and IP-headers
-                attacker_ether = Ether(src=mac_source_list[attacker], dst=mac_destination)
-                attacker_ip = IP(src=ip_source_list[attacker], dst=ip_destination, ttl=source_ttl_value, flags='DF')
-                victim_ether = Ether(src=mac_destination, dst=mac_source_list[attacker])
-                victim_ip = IP(src=ip_destination, dst=ip_source_list[attacker], ttl=destination_ttl_value, flags='DF')
-
-                # connection request from attacker (client)
-                syn_tcp = TCP(sport=sport, dport=smb_port, window=source_win_value, flags='S',
-                              seq=attacker_seq, options=[('MSS', source_mss_value)])
-                attacker_seq += 1
-                syn = (attacker_ether / attacker_ip / syn_tcp)
-                syn.time = timestamp_next_pkt
-                timestamp_next_pkt = update_timestamp(timestamp_next_pkt, victim_pps, minDelay)
-                packets.append(syn)
-
-                # response from victim (server)
-                synack_tcp = TCP(sport=smb_port, dport=sport, seq=victim_seq, ack=attacker_seq, flags='SA',
-                                 window=destination_win_value, options=[('MSS', destination_mss_value)])
-                victim_seq += 1
-                synack = (victim_ether / victim_ip / synack_tcp)
-                synack.time = timestamp_next_pkt
-                timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps, minDelay)
-                packets.append(synack)
-
-                # acknowledgement from attacker (client)
-                ack_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq, flags='A',
-                              window=source_win_value, options=[('MSS', source_mss_value)])
-                ack = (attacker_ether / attacker_ip / ack_tcp)
-                ack.time = timestamp_next_pkt
-                timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
-                packets.append(ack)
-
-                # send NBT session header paket with maximum LENGTH-field
-                req_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq, flags='AP',
-                              window=source_win_value, options=[('MSS', source_mss_value)])
-                req_payload = NBTSession(TYPE=0x00, LENGTH=0x1FFFF)
-
-                attacker_seq += len(req_payload)
-                req = (attacker_ether / attacker_ip / req_tcp / req_payload)
-                req.time = timestamp_next_pkt
-                timestamp_next_pkt = update_timestamp(timestamp_next_pkt, victim_pps, minDelay)
-                packets.append(req)
-
-                # final ack from victim (server)
-                last_ack_tcp = TCP(sport=smb_port, dport=sport, seq=victim_seq, ack=attacker_seq, flags='A',
-                                   window=destination_win_value, options=[('MSS', destination_mss_value)])
-                last_ack = (victim_ether / victim_ip / last_ack_tcp)
-                last_ack.time = timestamp_next_pkt
-                timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps, minDelay)
-                packets.append(last_ack)
-
-                sport += 1
-
-        # store end time of attack
-        self.attack_end_utime = packets[-1].time
-
-        # write attack packets to pcap
-        pcap_path = self.write_attack_pcap(sorted(packets, key=lambda pkt: pkt.time))
-
-        # return packets sorted by packet time_sec_start
-        return len(packets), pcap_path

code/Attack/SMBScanAttack.py

@@ -1,410 +0,0 @@
-import logging
-
-from random import shuffle, randint
-from lea import Lea
-from scapy.layers.inet import IP, Ether, TCP
-from scapy.layers.smb import *
-from scapy.layers.netbios import *
-
-from Attack import BaseAttack
-from Attack.AttackParameters import Parameter as Param
-from Attack.AttackParameters import ParameterTypes
-from ID2TLib.SMB2 import *
-from ID2TLib.Utility import update_timestamp, get_interval_pps, get_rnd_os, get_ip_range,\
-    generate_source_port_from_platform, get_filetime_format
-from ID2TLib.SMBLib import smb_port, smb_versions, smb_dialects, get_smb_version, get_smb_platform_data,\
-    invalid_smb_version
-
-logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
-# noinspection PyPep8
-
-
-class SMBScanAttack(BaseAttack.BaseAttack):
-
-    def __init__(self):
-        """
-        Creates a new instance of the SMBScanAttack.
-
-        """
-        # Initialize attack
-        super(SMBScanAttack, self).__init__("SmbScan Attack", "Injects an SMB scan",
-                                             "Scanning/Probing")
-
-        # Define allowed parameters and their type
-        self.supported_params = {
-            Param.IP_SOURCE: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.IP_DESTINATION: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.PORT_SOURCE: ParameterTypes.TYPE_PORT,
-            Param.MAC_SOURCE: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.MAC_DESTINATION: ParameterTypes.TYPE_MAC_ADDRESS,
-            Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
-            Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
-            Param.IP_SOURCE_RANDOMIZE: ParameterTypes.TYPE_BOOLEAN,
-            Param.PACKETS_PER_SECOND: ParameterTypes.TYPE_FLOAT,
-            Param.PORT_SOURCE_RANDOMIZE: ParameterTypes.TYPE_BOOLEAN,
-            Param.HOSTING_IP: ParameterTypes.TYPE_IP_ADDRESS,
-            Param.HOSTING_VERSION: ParameterTypes.TYPE_STRING,
-            Param.SOURCE_PLATFORM: ParameterTypes.TYPE_STRING,
-            Param.PROTOCOL_VERSION: ParameterTypes.TYPE_STRING,
-            Param.IP_DESTINATION_END: ParameterTypes.TYPE_IP_ADDRESS
-        }
-
-    def init_params(self):
-        """
-        Initialize the parameters of this attack using the user supplied command line parameters.
-        Use the provided statistics to calculate default parameters and to process user
-        supplied queries.
-
-        :param statistics: Reference to a statistics object.
-        """
-        # PARAMETERS: initialize with default values
-        # (values are overwritten if user specifies them)
-        most_used_ip_address = self.statistics.get_most_used_ip_address()
-        if isinstance(most_used_ip_address, list):
-            most_used_ip_address = most_used_ip_address[0]
-
-        self.add_param_value(Param.IP_SOURCE, most_used_ip_address)
-        self.add_param_value(Param.IP_SOURCE_RANDOMIZE, 'False')
-        self.add_param_value(Param.MAC_SOURCE, self.statistics.get_mac_address(most_used_ip_address))
-
-        all_ips = self.statistics.get_ip_addresses()
-        if not isinstance(all_ips, list):
-            ip_destinations = []
-            ip_destinations.append(all_ips)
-        else:
-            ip_destinations = all_ips
-        self.add_param_value(Param.IP_DESTINATION, ip_destinations)
-        destination_mac = []
-        for ip in ip_destinations:
-            destination_mac.append(self.statistics.get_mac_address(str(ip)))
-        if isinstance(destination_mac, list) and len(destination_mac) == 0:
-            destination_mac = self.generate_random_mac_address()
-        self.add_param_value(Param.MAC_DESTINATION, destination_mac)
-        self.add_param_value(Param.PORT_SOURCE, randint(1024, 65535))
-        self.add_param_value(Param.PORT_SOURCE_RANDOMIZE, 'True')
-        self.add_param_value(Param.PACKETS_PER_SECOND,
-                             (self.statistics.get_pps_sent(most_used_ip_address) +
-                              self.statistics.get_pps_received(most_used_ip_address)) / 2)
-        self.add_param_value(Param.INJECT_AFTER_PACKET, randint(0, self.statistics.get_packet_count()))
-
-        rnd_ip_count = self.statistics.get_ip_address_count()/2
-        self.add_param_value(Param.HOSTING_IP, self.statistics.get_random_ip_address(rnd_ip_count))
-        self.host_os = get_rnd_os()
-        self.add_param_value(Param.HOSTING_VERSION, get_smb_version(platform=self.host_os))
-        self.add_param_value(Param.SOURCE_PLATFORM, get_rnd_os())
-        self.add_param_value(Param.PROTOCOL_VERSION, "1")
-        self.add_param_value(Param.IP_DESTINATION_END, "0.0.0.0")
-
-    def generate_attack_pcap(self):
-        def get_ip_data(ip_address: str):
-            """
-            Gets the MSS, TTL and Windows Size values of a given IP
-
-            :param ip_address: the ip of which (packet-)data shall be returned
-            :return: MSS, TTL and Window Size values of the given IP
-            """
-            # Set MSS (Maximum Segment Size) based on MSS distribution of IP address
-            mss_dist = self.statistics.get_mss_distribution(ip_address)
-            if len(mss_dist) > 0:
-                mss_prob_dict = Lea.fromValFreqsDict(mss_dist)
-                mss_value = mss_prob_dict.random()
-            else:
-                mss_value = self.statistics.process_db_query("most_used(mssValue)")
-
-            # Set TTL based on TTL distribution of IP address
-            ttl_dist = self.statistics.get_ttl_distribution(ip_address)
-            if len(ttl_dist) > 0:
-                ttl_prob_dict = Lea.fromValFreqsDict(ttl_dist)
-                ttl_value = ttl_prob_dict.random()
-            else:
-                ttl_value = self.statistics.process_db_query("most_used(ttlValue)")
-
-            # Set Window Size based on Window Size distribution of IP address
-            win_dist = self.statistics.get_win_distribution(ip_address)
-            if len(win_dist) > 0:
-                win_prob_dict = Lea.fromValFreqsDict(win_dist)
-                win_value = win_prob_dict.random()
-            else:
-                win_value = self.statistics.process_db_query("most_used(winSize)")
-
-            return mss_value, ttl_value, win_value
-
-        pps = self.get_param_value(Param.PACKETS_PER_SECOND)
-
-        # Calculate complement packet rates of the background traffic for each interval
-        complement_interval_pps = self.statistics.calculate_complement_packet_rates(pps)
-
-
-        # Timestamp
-        timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
-        # store start time of attack
-        self.attack_start_utime = timestamp_next_pkt
-        timestamp_prv_reply, timestamp_confirm = 0,0
-
-        # Initialize parameters
-        ip_source = self.get_param_value(Param.IP_SOURCE)
-        ip_destinations = self.get_param_value(Param.IP_DESTINATION)
-        hosting_ip = self.get_param_value(Param.HOSTING_IP)
-        ip_range_end = self.get_param_value(Param.IP_DESTINATION_END)
-        mac_source = self.get_param_value(Param.MAC_SOURCE)
-        mac_dest = self.get_param_value(Param.MAC_DESTINATION)
-
-        # Check smb version
-        smb_version = self.get_param_value(Param.PROTOCOL_VERSION)
-        if smb_version not in smb_versions:
-            invalid_smb_version(smb_version)
-        hosting_version = self.get_param_value(Param.HOSTING_VERSION)
-        if hosting_version not in smb_versions:
-            invalid_smb_version(hosting_version)
-        # Check source platform
-        src_platform = self.get_param_value(Param.SOURCE_PLATFORM).lower()
-        packets = []
-
-        # randomize source ports according to platform, if specified
-        if self.get_param_value(Param.PORT_SOURCE_RANDOMIZE):
-            sport = generate_source_port_from_platform(src_platform)
-        else:
-            sport = self.get_param_value(Param.PORT_SOURCE)
-
-        # No destination IP was specified, but a destination MAC was specified, generate IP that fits MAC
-        if isinstance(ip_destinations, list) and isinstance(mac_dest, str):
-            ip_destinations = self.statistics.get_ip_address_from_mac(mac_dest)
-            if len(ip_destinations) == 0:
-                ip_destinations = self.generate_random_ipv4_address("Unknown", 1)
-            # Check ip.src == ip.dst
-            self.ip_src_dst_equal_check(ip_source, ip_destinations)
-
-        ip_dests = []
-        if isinstance(ip_destinations, list):
-            ip_dests = ip_destinations
-        else:
-            ip_dests.append(ip_destinations)
-
-        # Generate IPs of destination IP range, if specified
-        if ip_range_end != "0.0.0.0":
-            ip_dests = get_ip_range(ip_dests[0], ip_range_end)
-            shuffle(ip_dests)
-
-        # Randomize source IP, if specified
-        if self.get_param_value(Param.IP_SOURCE_RANDOMIZE):
-            ip_source = self.generate_random_ipv4_address("Unknown", 1)
-            while ip_source in ip_dests:
-                ip_source = self.generate_random_ipv4_address("Unknown", 1)
-            mac_source = self.statistics.get_mac_address(str(ip_source))
-            if len(mac_source) == 0:
-                mac_source = self.generate_random_mac_address()
-
-        # Get MSS, TTL and Window size value for source IP
-        source_mss_value, source_ttl_value, source_win_value = get_ip_data(ip_source)
-
-        for ip in ip_dests:
-
-            if ip != ip_source:
-
-                # Get destination Mac Address
-                mac_destination = self.statistics.get_mac_address(str(ip))
-                if len(mac_destination) == 0:
-                    if isinstance(mac_dest, str):
-                        if len(self.statistics.get_ip_address_from_mac(mac_dest)) != 0:
-                            ip = self.statistics.get_ip_address_from_mac(mac_dest)
-                            self.ip_src_dst_equal_check(ip_source, ip)
-
-                        mac_destination = mac_dest
-
-                    else:
-                        mac_destination = self.generate_random_mac_address()
-
-                # Get MSS, TTL and Window size value for destination IP
-                destination_mss_value, destination_ttl_value, destination_win_value = get_ip_data(ip)
-
-                minDelay, maxDelay = self.get_reply_delay(ip)
-
-                # New connection, new random TCP sequence numbers
-                attacker_seq = randint(1000, 50000)
-                victim_seq = randint(1000, 50000)
-
-                # Randomize source port for each connection if specified
-                if self.get_param_value(Param.PORT_SOURCE_RANDOMIZE):
-                    sport = generate_source_port_from_platform(src_platform, sport)
-
-                # 1) Build request package
-                request_ether = Ether(src=mac_source, dst=mac_destination)
-                request_ip = IP(src=ip_source, dst=ip, ttl=source_ttl_value, flags='DF')
-                request_tcp = TCP(sport=sport, dport=smb_port, window=source_win_value, flags='S',
-                                  seq=attacker_seq, options=[('MSS', source_mss_value)])
-                attacker_seq += 1
-                request = (request_ether / request_ip / request_tcp)
-                request.time = timestamp_next_pkt
-
-                # Append request
-                packets.append(request)
-
-                # Update timestamp for next package
-                timestamp_reply = update_timestamp(timestamp_next_pkt, pps, minDelay)
-                while (timestamp_reply <= timestamp_prv_reply):
-                    timestamp_reply = update_timestamp(timestamp_prv_reply, pps, minDelay)
-                timestamp_prv_reply = timestamp_reply
-
-                if ip in hosting_ip:
-
-                    # 2) Build TCP packages for ip that hosts SMB
-
-                    # destination sends SYN, ACK
-                    reply_ether = Ether(src=mac_destination, dst=mac_source)
-                    reply_ip = IP(src=ip, dst=ip_source, ttl=destination_ttl_value, flags='DF')
-                    reply_tcp = TCP(sport=smb_port, dport=sport, seq=victim_seq, ack=attacker_seq, flags='SA',
-                                    window=destination_win_value, options=[('MSS', destination_mss_value)])
-                    victim_seq += 1
-                    reply = (reply_ether / reply_ip / reply_tcp)
-                    reply.time = timestamp_reply
-                    packets.append(reply)
-
-                    # requester confirms, ACK
-                    confirm_ether = request_ether
-                    confirm_ip = request_ip
-                    confirm_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq,
-                                      window=source_win_value, flags='A')
-                    confirm = (confirm_ether / confirm_ip / confirm_tcp)
-                    timestamp_confirm = update_timestamp(timestamp_reply, pps, minDelay)
-                    confirm.time = timestamp_confirm
-                    packets.append(confirm)
-
-                    smb_MID = randint(1, 65535)
-                    smb_PID = randint(1, 65535)
-                    smb_req_tail_arr = []
-                    smb_req_tail_size = 0
-
-                    # select dialects based on smb version
-                    if smb_version is "1":
-                        smb_req_dialects = smb_dialects[0:6]
-                    else:
-                        smb_req_dialects = smb_dialects
-                    if len(smb_req_dialects) == 0:
-                        smb_req_tail_arr.append(SMBNegociate_Protocol_Request_Tail())
-                        smb_req_tail_size = len(SMBNegociate_Protocol_Request_Tail())
-                    else:
-                        for dia in smb_req_dialects:
-                            smb_req_tail_arr.append(SMBNegociate_Protocol_Request_Tail(BufferData = dia))
-                            smb_req_tail_size += len(SMBNegociate_Protocol_Request_Tail(BufferData = dia))
-
-                    smb_req_head = SMBNegociate_Protocol_Request_Header\
-                        (Flags2=0x2801, PID=smb_PID, MID=smb_MID, ByteCount=smb_req_tail_size)
-                    smb_req_length = len(smb_req_head) + smb_req_tail_size
-                    smb_req_net_bio = NBTSession(TYPE=0x00, LENGTH=smb_req_length)
-                    smb_req_tcp = TCP(sport=sport, dport=smb_port, flags='PA', seq=attacker_seq, ack=victim_seq)
-                    smb_req_ip = IP(src=ip_source, dst=ip, ttl=source_ttl_value)
-                    smb_req_ether = Ether(src=mac_source, dst=mac_destination)
-                    attacker_seq += len(smb_req_net_bio) + len(smb_req_head) + smb_req_tail_size
-
-                    smb_req_combined = (smb_req_ether / smb_req_ip / smb_req_tcp / smb_req_net_bio / smb_req_head)
-
-                    for i in range(0 , len(smb_req_tail_arr)):
-                        smb_req_combined = smb_req_combined / smb_req_tail_arr[i]
-
-                    timestamp_smb_req = update_timestamp(timestamp_confirm, pps, minDelay)
-                    smb_req_combined.time = timestamp_smb_req
-                    packets.append(smb_req_combined)
-
-                    # destination confirms SMB request package
-                    reply_tcp = TCP(sport=smb_port, dport=sport, seq=victim_seq, ack=attacker_seq,
-                                    window=destination_win_value, flags='A')
-                    confirm_smb_req = (reply_ether / reply_ip / reply_tcp)
-                    timestamp_reply = update_timestamp(timestamp_smb_req, pps, minDelay)
-                    confirm_smb_req.time = timestamp_reply
-                    packets.append(confirm_smb_req)
-
-                    # smb response package
-                    first_timestamp = time.mktime(time.strptime(self.statistics.get_pcap_timestamp_start()[:19],
-                                                                "%Y-%m-%d %H:%M:%S"))
-                    server_Guid, security_blob, capabilities, data_size, server_start_time = get_smb_platform_data\
-                        (self.host_os, first_timestamp)
-
-                    timestamp_smb_rsp = update_timestamp(timestamp_reply, pps, minDelay)
-                    diff = timestamp_smb_rsp - timestamp_smb_req
-                    begin = get_filetime_format(timestamp_smb_req+diff*0.1)
-                    end = get_filetime_format(timestamp_smb_rsp-diff*0.1)
-                    system_time = randint(begin, end)
-
-                    if smb_version is not "1" and hosting_version is not "1":
-                        smb_rsp_paket = SMB2_SYNC_Header(Flags = 1)
-                        smb_rsp_negotiate_body = SMB2_Negotiate_Protocol_Response\
-                            (DialectRevision=0x02ff, SecurityBufferOffset=124, SecurityBufferLength=len(security_blob),
-                             SecurityBlob=security_blob, Capabilities=capabilities, MaxTransactSize=data_size,
-                             MaxReadSize=data_size, MaxWriteSize=data_size, SystemTime=system_time,
-                             ServerStartTime=server_start_time, ServerGuid=server_Guid)
-                        smb_rsp_length = len(smb_rsp_paket) + len(smb_rsp_negotiate_body)
-                    else:
-                        smb_rsp_paket = SMBNegociate_Protocol_Response_Advanced_Security\
-                            (Start="\xffSMB", PID=smb_PID, MID=smb_MID, DialectIndex=5, SecurityBlob=security_blob)
-                        smb_rsp_length = len(smb_rsp_paket)
-                    smb_rsp_net_bio = NBTSession(TYPE=0x00, LENGTH=smb_rsp_length)
-                    smb_rsp_tcp = TCP(sport=smb_port, dport=sport, flags='PA', seq=victim_seq, ack=attacker_seq)
-                    smb_rsp_ip = IP(src=ip, dst=ip_source, ttl=destination_ttl_value)
-                    smb_rsp_ether = Ether(src=mac_destination, dst=mac_source)
-                    victim_seq += len(smb_rsp_net_bio) + len(smb_rsp_paket)
-                    if smb_version is not "1"and hosting_version is not "1":
-                        victim_seq += len(smb_rsp_negotiate_body)
-
-                    smb_rsp_combined = (smb_rsp_ether / smb_rsp_ip / smb_rsp_tcp / smb_rsp_net_bio / smb_rsp_paket)
-                    if smb_version is not "1"and hosting_version is not "1":
-                        smb_rsp_combined = (smb_rsp_combined / smb_rsp_negotiate_body)
-
-                    smb_rsp_combined.time = timestamp_smb_rsp
-                    packets.append(smb_rsp_combined)
-
-
-                    # source confirms SMB response package
-                    confirm_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq,
-                                      window=source_win_value, flags='A')
-                    confirm_smb_res = (confirm_ether / confirm_ip / confirm_tcp)
-                    timestamp_confirm = update_timestamp(timestamp_smb_rsp, pps, minDelay)
-                    confirm_smb_res.time = timestamp_confirm
-                    packets.append(confirm_smb_res)
-
-                    # attacker sends FIN ACK
-                    confirm_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq,
-                                      window=source_win_value, flags='FA')
-                    source_fin_ack = (confirm_ether / confirm_ip / confirm_tcp)
-                    timestamp_src_fin_ack = update_timestamp(timestamp_confirm, pps, minDelay)
-                    source_fin_ack.time = timestamp_src_fin_ack
-                    attacker_seq += 1
-                    packets.append(source_fin_ack)
-
-                    # victim sends FIN ACK
-                    reply_tcp = TCP(sport=smb_port, dport=sport, seq=victim_seq, ack=attacker_seq,
-                                    window=destination_win_value, flags='FA')
-                    destination_fin_ack = (reply_ether / reply_ip / reply_tcp)
-                    timestamp_dest_fin_ack = update_timestamp(timestamp_src_fin_ack, pps, minDelay)
-                    victim_seq += 1
-                    destination_fin_ack.time = timestamp_dest_fin_ack
-                    packets.append(destination_fin_ack)
-
-                    # source sends final ACK
-                    confirm_tcp = TCP(sport=sport, dport=smb_port, seq=attacker_seq, ack=victim_seq,
-                                      window=source_win_value, flags='A')
-                    final_ack = (confirm_ether / confirm_ip / confirm_tcp)
-                    timestamp_final_ack = update_timestamp(timestamp_dest_fin_ack, pps, minDelay)
-                    final_ack.time = timestamp_final_ack
-                    packets.append(final_ack)
-
-                else:
-                    # Build RST package
-                    reply_ether = Ether(src=mac_destination, dst=mac_source)
-                    reply_ip = IP(src=ip, dst=ip_source, ttl=destination_ttl_value, flags='DF')
-                    reply_tcp = TCP(sport=smb_port, dport=sport, seq=0, ack=attacker_seq, flags='RA',
-                                    window=destination_win_value, options=[('MSS', destination_mss_value)])
-                    reply = (reply_ether / reply_ip / reply_tcp)
-                    reply.time = timestamp_reply
-                    packets.append(reply)
-
-            pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
-            timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
-
-        # store end time of attack
-        self.attack_end_utime = packets[-1].time
-
-        # write attack packets to pcap
-        pcap_path = self.write_attack_pcap(sorted(packets, key=lambda pkt: pkt.time))
-
-        # return packets sorted by packet time_sec_start
-        return len(packets), pcap_path

code/Attack/SQLiAttack.py

@@ -1,10 +1,7 @@
 import logging
+from random import randint, uniform
 
-from random import randint
 from lea import Lea
-from scapy.utils import RawPcapReader
-from scapy.layers.inet import Ether
-from ID2TLib.Utility import update_timestamp, get_interval_pps
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
@@ -12,6 +9,8 @@ from Attack.AttackParameters import ParameterTypes
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
+from scapy.utils import RawPcapReader
+from scapy.layers.inet import IP, Ether, TCP, RandShort
 
 
 class SQLiAttack(BaseAttack.BaseAttack):
@@ -79,7 +78,29 @@ class SQLiAttack(BaseAttack.BaseAttack):
                              (self.statistics.get_pps_sent(most_used_ip_address) +
                               self.statistics.get_pps_received(most_used_ip_address)) / 2)
 
-    def generate_attack_pcap(self):
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            # Calculate the request timestamp
+            # A distribution to imitate the bursty behavior of traffic
+            randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+            return timestamp + uniform(1 / pps, randomdelay.random())
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) in specific time interval.
+
+            :return: the corresponding packet rate for packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp <= row[0]:
+                    return row[1]
+            return complement_interval_pps[-1][1]  # in case the timstamp > capture max timestamp
+
         # Timestamp
         timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
         pps = self.get_param_value(Param.PACKETS_PER_SECOND)
@@ -187,7 +208,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
                     new_pkt = (eth_frame / ip_pkt/ tcp_pkt / str_tcp_seg)
                     new_pkt.time = timestamp_next_pkt
 
-                    pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                    pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                     timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + float(timeSteps.random())
 
                 # Victim --> attacker
@@ -249,7 +270,7 @@ class SQLiAttack(BaseAttack.BaseAttack):
                     new_pkt = (eth_frame / ip_pkt / tcp_pkt / str_tcp_seg)
                     new_pkt.time = timestamp_next_pkt
 
-                    pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+                    pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
                     timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps) + float(timeSteps.random())
 
                 # Victim --> attacker

code/Attack/SalityBotnet.py

@@ -1,16 +1,16 @@
 import logging
+from random import randint, uniform
 
-from random import randint
-from scapy.utils import RawPcapReader
-from scapy.layers.inet import Ether
+from lea import Lea
 
 from Attack import BaseAttack
 from Attack.AttackParameters import Parameter as Param
 from Attack.AttackParameters import ParameterTypes
-from ID2TLib.Utility import update_timestamp, get_interval_pps
 
 logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
 # noinspection PyPep8
+from scapy.utils import RawPcapReader
+from scapy.layers.inet import IP, Ether, TCP, RandShort
 
 
 class SalityBotnet(BaseAttack.BaseAttack):
@@ -56,7 +56,28 @@ class SalityBotnet(BaseAttack.BaseAttack):
                              (self.statistics.get_pps_sent(most_used_ip_address) +
                               self.statistics.get_pps_received(most_used_ip_address)) / 2)
 
-    def generate_attack_pcap(self):
+    def generate_attack_pcap(self, context):
+        def update_timestamp(timestamp, pps):
+            """
+            Calculates the next timestamp to be used based on the packet per second rate (pps) and the maximum delay.
+
+            :return: Timestamp to be used for the next packet.
+            """
+            # Calculate the request timestamp
+            # A distribution to imitate the bursty behavior of traffic
+            randomdelay = Lea.fromValFreqsDict({1 / pps: 70, 2 / pps: 20, 5 / pps: 7, 10 / pps: 3})
+            return timestamp + uniform(1 / pps, randomdelay.random())
+
+        def getIntervalPPS(complement_interval_pps, timestamp):
+            """
+            Gets the packet rate (pps) in specific time interval.
+
+            :return: the corresponding packet rate for packet rate (pps) .
+            """
+            for row in complement_interval_pps:
+                if timestamp <= row[0]:
+                    return row[1]
+            return complement_interval_pps[-1][1]  # in case the timstamp > capture max timestamp
 
         # Timestamp
         timestamp_next_pkt = self.get_param_value(Param.INJECT_AT_TIMESTAMP)
@@ -122,7 +143,7 @@ class SalityBotnet(BaseAttack.BaseAttack):
             new_pkt = (eth_frame / ip_pkt)
             new_pkt.time = timestamp_next_pkt
 
-            pps = max(get_interval_pps(complement_interval_pps, timestamp_next_pkt), 10)
+            pps = max(getIntervalPPS(complement_interval_pps, timestamp_next_pkt), 10)
             timestamp_next_pkt = update_timestamp(timestamp_next_pkt, pps)
 
             packets.append(new_pkt)

code/ID2TLib/AttackController.py

@@ -4,7 +4,6 @@ import sys
 from Attack.AttackParameters import Parameter
 from ID2TLib import LabelManager
 from ID2TLib import Statistics
-from ID2TLib.Label import Label
 from ID2TLib.PcapFile import PcapFile
 
 
@@ -40,7 +39,7 @@ class AttackController:
         # Record the attack
         self.added_attacks.append(self.current_attack)
 
-    def process_attack(self, attack: str, params: str):
+    def process_attack(self, attack: str, params: str, context):
         """
         Takes as input the name of an attack (classname) and the attack parameters as string. Parses the string of
         attack parameters, creates the attack by writing the attack packets and returns the path of the written pcap.
@@ -81,11 +80,11 @@ class AttackController:
         # Write attack into pcap file
         print("Generating attack packets...", end=" ")
         sys.stdout.flush()  # force python to print text immediately
-        total_packets, temp_attack_pcap_path = self.current_attack.generate_attack_pcap()
+        total_packets, temp_attack_pcap_path = self.current_attack.generate_attack_pcap(context)
         print("done. (total: " + str(total_packets) + " pkts.)")
 
         # Store label into LabelManager
-        l = Label(attack, self.get_attack_start_utime(),
+        l = LabelManager.Label(attack, self.get_attack_start_utime(),
                   self.get_attack_end_utime(), attack_note)
         self.label_mgr.add_labels(l)
 

code/ID2TLib/Controller.py

@@ -1,21 +1,26 @@
 import os
 import sys
-import readline
+import shutil
 
 from ID2TLib.AttackController import AttackController
 from ID2TLib.LabelManager import LabelManager
 from ID2TLib.PcapFile import PcapFile
 from ID2TLib.Statistics import Statistics
+from ID2TLib.AttackContext import AttackContext
 
 
 class Controller:
-    def __init__(self, pcap_file_path: str, do_extra_tests: bool):
+    def __init__(self, in_pcap_file_path: str, do_extra_tests: bool, out_pcap_file_path):
         """
         Creates a new Controller, acting as a central coordinator for the whole application.
         :param pcap_file_path:
         """
         # Fields
-        self.pcap_src_path = pcap_file_path.strip()
+        self.pcap_src_path = in_pcap_file_path.strip()
+        if out_pcap_file_path:
+            self.pcap_out_path = out_pcap_file_path.strip()
+        else:
+            self.pcap_out_path = None
         self.pcap_dest_path = ''
         self.written_pcaps = []
         self.do_extra_tests = do_extra_tests
@@ -49,9 +54,23 @@ class Controller:
         input dataset.
         :param attacks_config: A list of attacks with their attack parameters.
         """
+
+        # get output directory
+        if self.pcap_out_path:
+            out_dir = os.path.dirname(self.pcap_out_path)
+        else:
+            out_dir = os.path.dirname(self.pcap_src_path)
+        # if out_dir is cwd
+        if out_dir == "":
+            out_dir = "."
+
+        # context for the attack(s)
+        context = AttackContext(out_dir)
+
+        # note if new xml file has been created by MembersMgmtCommAttack
         # load attacks sequentially
         for attack in attacks_config:
-            temp_attack_pcap = self.attack_controller.process_attack(attack[0], attack[1:])
+            temp_attack_pcap = self.attack_controller.process_attack(attack[0], attack[1:], context)
             self.written_pcaps.append(temp_attack_pcap)
 
         # merge attack pcaps to get single attack pcap
@@ -62,7 +81,6 @@ class Controller:
                 attacks_pcap = PcapFile(self.written_pcaps[i])
                 attacks_pcap_path = attacks_pcap.merge_attack(self.written_pcaps[i + 1])
                 os.remove(self.written_pcaps[i + 1])  # remove merged pcap
-                self.written_pcaps[i + 1] = attacks_pcap_path
             print("done.")
         else:
             attacks_pcap_path = self.written_pcaps[0]
@@ -70,7 +88,15 @@ class Controller:
         # merge single attack pcap with all attacks into base pcap
         print("Merging base pcap with single attack pcap...", end=" ")
         sys.stdout.flush()  # force python to print text immediately
+
+        # cp merged PCAP to output path
         self.pcap_dest_path = self.pcap_file.merge_attack(attacks_pcap_path)
+        if self.pcap_out_path:
+            if not self.pcap_out_path.endswith(".pcap"):
+                self.pcap_out_path += ".pcap"
+            os.rename(self.pcap_dest_path, self.pcap_out_path)
+            self.pcap_dest_path = self.pcap_out_path
+
         print("done.")
 
         # delete intermediate PCAP files
@@ -82,8 +108,23 @@ class Controller:
         # write label file with attacks
         self.label_manager.write_label_file(self.pcap_dest_path)
 
+        # pcap_base contains the name of the pcap-file without the ".pcap" extension
+        pcap_base = os.path.splitext(self.pcap_dest_path)[0]
+        created_files = [self.pcap_dest_path, self.label_manager.label_file_path]
+        for suffix, filename in context.get_allocated_files():
+            shutil.move(filename, pcap_base + suffix)
+            created_files.append(pcap_base + suffix)
+        context.reset()
+
         # print status message
-        print('\nOutput files created: \n', self.pcap_dest_path, '\n', self.label_manager.label_file_path)
+        created_files += context.get_other_created_files()
+        created_files.sort()
+        print("\nOutput files created:")
+        for file in created_files:
+            # remove ./ at beginning of file to have only one representation for cwd
+            if file.startswith("./"):
+                file = file[2:]
+            print(file)
 
     def process_db_queries(self, query, print_results=False):
         """
@@ -99,109 +140,13 @@ class Controller:
         else:
             self.statisticsDB.process_db_query(query, print_results)
 
-    @staticmethod
-    def process_help(params):
-        if not params:
-            print("Query mode allows you to enter SQL-queries as well as named queries.")
-            print()
-            print("Named queries:")
-            print("\tSelectors:")
-            print("\t\tmost_used(...)  -> Returns the most occurring element in all elements")
-            print("\t\tleast_used(...) -> Returns the least occurring element in all elements")
-            print("\t\tavg(...)        -> Returns the average of all elements")
-            print("\t\tall(...)        -> Returns all elements")
-            print("\tExtractors:")
-            print("\t\trandom(...)     -> Returns a random element from a list")
-            print("\t\tfirst(...)      -> Returns the first element from a list")
-            print("\t\tlast(...)       -> Returns the last element from a list")
-            print("\tParameterized selectors:")
-            print("\t\tipAddress(...)  -> Returns all IP addresses fulfilling the specified conditions")
-            print("\t\tmacAddress(...) -> Returns all MAC addresses fulfilling the specified conditions")
-            print()
-            print("Miscellaneous:")
-            print("\tlabels            -> List all attacks listed in the label file, if any")
-            print()
-            print("Additional information is available with 'help [KEYWORD];'")
-            print("To get a list of examples, type 'help examples;'")
-            print()
-            return
-
-        param = params[0].lower()
-        if param == "most_used":
-            print("most_used can be used as a selector for the following attributes:")
-            print("ipAddress | macAddress | portNumber | protocolName | ttlValue | mssValue | winSize | ipClass")
-            print()
-        elif param == "least_used":
-            print("least_used can be used as a selector for the following attributes:")
-            print("ipAddress | macAddress | portNumber | protocolName | ttlValue")
-            print()
-        elif param == "avg":
-            print("avg can be used as a selector for the following attributes:")
-            print("pktsReceived | pktsSent | kbytesSent | kbytesReceived | ttlValue | mss")
-            print()
-        elif param == "all":
-            print("all can be used as a selector for the following attributes:")
-            print("ipAddress | ttlValue | mss | macAddress | portNumber | protocolName")
-            print()
-        elif param in ["random", "first", "last"]:
-            print("No additional info available for this keyword.")
-            print()
-        elif param == "ipaddress":
-            print("ipAddress is a parameterized selector which fetches IP addresses based on (a list of) conditions.")
-            print("Conditions are of the following form: PARAMETER OPERATOR VALUE")
-            print("The following parameters can be specified:")
-            print("pktsReceived | pktsSent | kbytesReceived | kbytesSent | maxPktRate | minPktRate | ipClass\n"
-                  "macAddress | ttlValue | ttlCount | portDirection | portNumber | portCount | protocolCount\n"
-                  "protocolName")
-            print()
-            print("See 'help examples;' for usage examples.")
-            print()
-        elif param == "macaddress":
-            print("macAddress is a parameterized selector which fetches MAC addresses based on (a list of) conditions.")
-            print("Conditions are of the following form: PARAMETER OPERATOR VALUE")
-            print("The following parameters can be specified:")
-            print("ipAddress")
-            print()
-            print("See 'help examples;' for usage examples.")
-            print()
-        elif param == "examples":
-            print("Get the average amount of sent packets per IP:")
-            print("\tavg(pktsSent);")
-            print("Get a random IP from all addresses occuring in the pcap:")
-            print("\trandom(all(ipAddress));")
-            print("Return the MAC address of a specified IP:")
-            print("\tmacAddress(ipAddress=192.168.178.2);")
-            print("Get the average TTL-value with SQL:")
-            print("\tSELECT avg(ttlValue) from ip_ttl;")
-            print("Get a random IP address from all addresses that sent and received at least 10 packets:")
-            print("\trandom(ipAddress(pktsSent > 10, pktsReceived > 10));")
-            print()
-        else:
-            print("Unknown keyword '" + param + "', try 'help;' to get a list of allowed keywords'")
-            print()
-
     def enter_query_mode(self):
         """
         Enters into the query mode. This is a read-eval-print-loop, where the user can input named queries or SQL
         queries and the results are printed.
         """
-
-        def make_completer(vocabulary):
-            def custom_template(text, state):
-                results = [x for x in vocabulary if x.startswith(text)] + [None]
-                return results[state]
-            return custom_template
-
-        readline.parse_and_bind('tab: complete')
-        readline.set_completer(make_completer(self.statisticsDB.get_all_named_query_keywords()+self.statisticsDB.get_all_sql_query_keywords()))
-        history_file = os.path.join(os.path.expanduser('~'), 'ID2T_data', 'query_history')
-        try:
-            readline.read_history_file(history_file)
-        except IOError:
-            pass
         print("Entering into query mode...")
-        print("Enter statement ending by ';' and press ENTER to send query. Exit by sending an empty query.")
-        print("Type 'help;' for information on possible queries.")
+        print("Enter statement ending by ';' and press ENTER to send query. Exit by sending an empty query..")
         buffer = ""
         while True:
             line = input("> ")
@@ -212,31 +157,11 @@ class Controller:
             if sqlite3.complete_statement(buffer):
                 try:
                     buffer = buffer.strip()
-                    if buffer.lower().startswith('help'):
-                        buffer = buffer.strip(';')
-                        self.process_help(buffer.split(' ')[1:])
-                    elif buffer.lower().strip() == 'labels;':
-                        if not self.label_manager.labels:
-                            print("No labels found.")
-                        else:
-                            print("Attacks listed in the label file:")
-                            print()
-                            for label in self.label_manager.labels:
-                                print("Attack name:     " + str(label.attack_name))
-                                print("Attack note:     " + str(label.attack_note))
-                                print("Start timestamp: " + str(label.timestamp_start))
-                                print("End timestamp:   " + str(label.timestamp_end))
-                                print()
-                        print()
-                    else:
-                        self.statisticsDB.process_db_query(buffer, True)
+                    self.statisticsDB.process_db_query(buffer, True)
                 except sqlite3.Error as e:
                     print("An error occurred:", e.args[0])
                 buffer = ""
 
-        readline.set_history_length(1000)
-        readline.write_history_file(history_file)
-
     def create_statistics_plot(self, params: str):
         """
         Plots the statistics to a file by using the given customization parameters.

code/ID2TLib/LabelManager.py

@@ -2,7 +2,38 @@ import os.path
 from datetime import datetime
 from xml.dom.minidom import *
 
-import ID2TLib.Label as Label
+from functools import total_ordering
+
+@total_ordering
+class Label:
+    def __init__(self, attack_name, timestamp_start, timestamp_end, attack_note=""):
+        """
+        Creates a new attack label
+
+        :param attack_name: The name of the associated attack
+        :param timestamp_start: The timestamp as unix time of the first attack packet
+        :param timestamp_end: The timestamp as unix time of the last attack packet
+        :param attack_note: A note associated to the attack (optional)
+        """
+        self.attack_name = attack_name
+        self.timestamp_start = timestamp_start
+        self.timestamp_end = timestamp_end
+        self.attack_note = attack_note
+
+    def __eq__(self, other):
+        return self.timestamp == other.timestamp
+
+    def __lt__(self, other):
+        return self.timestamp_start < other.timestamp_start
+
+    def __gt__(self, other):
+        return self.timestamp_start > other.timestamp_start
+
+    def __str__(self):
+        return ''.join(
+            ['(', self.attack_name, ',', self.attack_note, ',', str(self.timestamp_start), ',', str(self.timestamp_end),
+             ')'])
+
 
 
 class LabelManager:
@@ -28,7 +59,8 @@ class LabelManager:
         self.labels = list()
 
         if filepath_pcap is not None:
-            self.label_file_path = filepath_pcap.strip('.pcap') + '_labels.xml'
+            # splitext gives us the filename without extension
+            self.label_file_path = os.path.splitext(filepath_pcap)[0] + '_labels.xml'
             # only load labels if label file is existing
             if os.path.exists(self.label_file_path):
                 self.load_labels()
@@ -83,7 +115,7 @@ class LabelManager:
             return timestamp_root
 
         if filepath is not None:
-            self.label_file_path = filepath.strip('.pcap') + '_labels.xml'
+            self.label_file_path = os.path.splitext(filepath)[0] + '_labels.xml' # splitext removes the file extension
 
         # Generate XML
         doc = Document()
@@ -162,7 +194,7 @@ class LabelManager:
             attack_note = get_value_from_node(a, self.TAG_ATTACK_NOTE, 0)
             timestamp_start = get_value_from_node(a, self.TAG_TIMESTAMP_START, 1, 0)
             timestamp_end = get_value_from_node(a, self.TAG_TIMESTAMP_END, 1, 0)
-            label = Label.Label(attack_name, float(timestamp_start), float(timestamp_end), attack_note)
+            label = Label(attack_name, float(timestamp_start), float(timestamp_end), attack_note)
             self.labels.append(label)
             count_labels += 1
 

code/ID2TLib/OldLibs/IPGenerator.py

@@ -0,0 +1,109 @@
+import random
+from .. import IPv4 as ip
+
+
+class IPChooser:
+	def random_ip(self) -> ip.IPAddress:
+		return ip.IPAddress.from_int(random.randrange(0, 1 << 32))
+	
+	def size(self) -> int:
+		return 1 << 32
+	
+	def __len__(self) -> int:
+		return self.size()
+
+class IPChooserByRange(IPChooser):
+	def __init__(self, ip_range: ip.IPAddressBlock) -> "IPChooserByRange":
+		self.range = ip_range
+	
+	def random_ip(self) -> ip.IPAddress:
+		start = int(self.range.first_address())
+		end = start + self.range.block_size()
+		return ip.IPAddress.from_int(random.randrange(start, end))
+	
+	def size(self) -> int:
+		return self.range.block_size()
+
+class IPChooserByList(IPChooser):
+	def __init__(self, ips: "list[ip.IPAddress]") -> "IPChooserByList":
+		self.ips = list(ips)
+		if not self.ips:
+			raise ValueError("list of ips must not be empty")
+	
+	def random_ip(self) -> ip.IPAddress:
+		return random.choice(self.ips)
+	
+	def size(self) -> int:
+		return len(self.ips)
+
+class IPGenerator:
+	def __init__(self, ip_chooser = IPChooser(), # include all ip-addresses by default (before the blacklist)
+			include_private_ips = False, include_localhost = False,
+			include_multicast = False, include_reserved = False,
+			include_link_local = False, blacklist = None) -> "IPGenerator":
+		self.blacklist = []
+		self.generated_ips = set()
+		
+		if not include_private_ips:
+			for segment in ip.ReservedIPBlocks.PRIVATE_IP_SEGMENTS:
+				self.add_to_blacklist(segment)
+		if not include_localhost:
+			self.add_to_blacklist(ip.ReservedIPBlocks.LOCALHOST_SEGMENT)
+		if not include_multicast:
+			self.add_to_blacklist(ip.ReservedIPBlocks.MULTICAST_SEGMENT)
+		if not include_reserved:
+			self.add_to_blacklist(ip.ReservedIPBlocks.RESERVED_SEGMENT)
+		if not include_link_local:
+			self.add_to_blacklist(ip.ReservedIPBlocks.ZERO_CONF_SEGMENT)
+		if blacklist:
+			for segment in blacklist:
+				self.add_to_blacklist(segment)
+		self.chooser = ip_chooser
+	
+	@staticmethod
+	def from_range(range: ip.IPAddressBlock, *args, **kwargs) -> "IPGenerator":
+		return IPGenerator(IPChooserByRange(range), *args, **kwargs)
+	
+	def add_to_blacklist(self, ip_segment: "Union[ip.IPAddressBlock, str]"):
+		if isinstance(ip_segment, ip.IPAddressBlock):
+			self.blacklist.append(ip_segment)
+		else:
+			self.blacklist.append(ip.IPAddressBlock.parse(ip_segment))
+	
+	def random_ip(self) -> ip.IPAddress:
+		if len(self.generated_ips) == self.chooser.size():
+			raise ValueError("Exhausted the space of possible ip-addresses, no new unique ip-address can be generated")
+		
+		while True:
+			random_ip = self.chooser.random_ip()
+			
+			if not self._is_in_blacklist(random_ip) and random_ip not in self.generated_ips:
+				self.generated_ips.add(random_ip)
+				return str(random_ip)
+	
+	def clear(self, clear_blacklist = True, clear_generated_ips = True):
+		if clear_blacklist: self.blacklist.clear()
+		if clear_generated_ips: self.generated_ips.clear()
+	
+	def _is_in_blacklist(self, ip: ip.IPAddress) -> bool:
+		return any(ip in block for block in self.blacklist)
+
+class MappingIPGenerator(IPGenerator):
+	def __init__(self, *args, **kwargs) -> "MappingIPGenerator":
+		super().__init__(self, *args, **kwargs)
+		
+		self.mapping = {}
+	
+	def clear(self, clear_generated_ips = True, *args, **kwargs):
+		super().clear(self, clear_generated_ips = clear_generated_ips, *args, **kwargs)
+		if clear_generated_ips:
+			self.mapping  = {}
+	
+	def get_mapped_ip(self, key) -> ip.IPAddress:
+		if key not in self.mapping:
+			self.mapping[key] = self.random_ip()
+		
+		return self.mapping[key]
+	
+	def __getitem__(self, item) -> ip.IPAddress:
+		return self.get_mapped_ip(item)

code/ID2TLib/OldLibs/MacAddressGenerator.py

@@ -0,0 +1,40 @@
+from random import getrandbits
+
+class MacAddressGenerator:
+	def __init__(self, include_broadcast_macs = False, include_virtual_macs = False) -> "MacAddressGenerator":
+		self.broadcast = include_broadcast_macs
+		self.virtual = include_virtual_macs
+		
+		self.generated = set()
+	
+	def random_mac(self) -> str:
+		while True:
+			mac = self._random_mac()
+			if mac not in self.generated:
+				self.generated.add(mac)
+				return mac
+	
+	def clear(self):
+		self.generated.clear()
+	
+	def generates_broadcast_macs(self) -> bool:
+		return self.broadcast
+	
+	def generates_virtual_macs(self) -> bool:
+		return self.virtual
+	
+	def set_broadcast_generation(self, broadcast: bool):
+		self.broadcast = broadcast
+	
+	def set_virtual_generation(self, virtual: bool):
+		self.virtual = virtual
+	
+	def _random_mac(self) -> str:
+		mac_bytes = bytearray(getrandbits(8) for i in range(6))
+		if not self.broadcast:
+			mac_bytes[0] &= ~1 # clear the first bytes' first bit
+		if not self.virtual:
+			mac_bytes[0] &= ~2 # clear the first bytes' second bit
+		
+		return ":".join("%02X" % b for b in mac_bytes)
+

code/ID2TLib/OldLibs/PacketGenerator.py

@@ -0,0 +1,137 @@
+from Attack.MembersMgmtCommAttack import MessageType
+
+import ID2TLib.PayloadGenerator as PayloadGenerator
+
+from scapy.layers.inet import IP, Ether, UDP, TCP
+from scapy.packet import Raw
+
+class PacketGenerator():
+	"""
+	Creates packets, based on the set protocol
+	"""
+	def __init__(self, protocol="udp"):
+		"""
+		Creates a new Packet_Generator Object
+		
+		:param protocol: the protocol of the packets to be created, udp or tcp
+		"""
+		super(PacketGenerator, self).__init__()
+		self.protocol = protocol
+
+	def generate_packet(self, ip_src: str="192.168.64.32", ip_dst: str="192.168.64.48", mac_src: str="56:6D:D9:BC:70:1C",
+		mac_dst: str="F4:2B:95:B3:0E:1A", port_src: int=1337, port_dst: int=6442, ttl: int=64,
+		tcpflags: str="S", payload: str=""):
+		"""
+		Creates a Packet with the specified Values for the current protocol
+
+		:param ip_src: the source IP address of the IP header
+	    :param ip_dst the destination IP address of the IP header
+	    :param mac_src: the source MAC address of the MAC header
+	    :param mac_dst: the destination MAC address of the MAC header
+	    :param port_src: the source port of the header
+	    :param port_dst: the destination port of the header
+	    :param ttl: the ttl Value of the packet
+	    :param tcpflags: the TCP flags of the TCP header
+	    :param payload: the payload of the packet
+	    :return: the corresponding packet
+	    """
+
+		if(self.protocol == "udp"):
+			packet = generate_udp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+				port_src=port_src, port_dst=port_dst, payload=payload)
+		elif(self.protocol == "tcp"):
+			packet = generate_tcp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+				port_src=port_src, port_dst=port_dst, tcpflags=tcpflags, payload=payload)
+		return packet
+		
+
+	def generate_mmcom_packet(self, ip_src: str="192.168.64.32", ip_dst: str="192.168.64.48", mac_src: str="56:6D:D9:BC:70:1C",
+			mac_dst: str="F4:2B:95:B3:0E:1A", port_src: int=1337, port_dst: int=6442, tcpflags: str="S", ttl: int=64,
+			message_type: MessageType=MessageType.SALITY_HELLO, neighborlist_entries: int=1):
+		"""
+		Creates a Packet for Members-Management-Communication with the specified Values and the current protocol
+
+		:param ip_src: the source IP address of the IP header
+	    :param ip_dst the destination IP address of the IP header
+	    :param mac_src: the source MAC address of the MAC header
+	    :param mac_dst: the destination MAC address of the MAC header
+	    :param port_src: the source port of the header
+	    :param port_dst: the destination port of the header
+	    :param tcpflags: the TCP flags of the TCP header, if tcp is selected as protocol
+	    :param ttl: the ttl Value of the packet
+	    :param message_type: affects the size of the payload
+	    :param neighborlist_entries: number of entries of a Neighbourlist-reply, affects the size of the payload
+	    :return: the corresponding packet
+	    """
+
+	    #Determine length of the payload that has to be generated
+		if(message_type == MessageType.SALITY_HELLO):
+			payload_len = 0
+		elif(message_type == MessageType.SALITY_HELLO_REPLY):
+			payload_len = 22
+		elif(message_type == MessageType.SALITY_NL_REQUEST):
+			payload_len = 28
+		elif(message_type == MessageType.SALITY_NL_REPLY):
+			payload_len = 24 + 6 * neighborlist_entries
+		else:
+			payload_len = 0
+
+		payload = PayloadGenerator.generate_payload(payload_len)
+
+		if(self.protocol == "udp"):
+			packet = generate_udp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+				port_src=port_src, port_dst=port_dst, payload=payload)
+		elif(self.protocol == "tcp"):
+			packet = generate_tcp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+				port_src=port_src, port_dst=port_dst, tcpflags=tcpflags, payload=payload)
+		else:
+			print("Error: unsupported protocol for generating Packets")
+
+		return packet
+
+
+def generate_tcp_packet(ip_src:str="192.168.64.32", ip_dst:str="192.168.64.48", mac_src:str="56:6D:D9:BC:70:1C", ttl: int=64,
+					   mac_dst:str="F4:2B:95:B3:0E:1A", port_src:int=1337, port_dst:int=6442, tcpflags:str="S", payload:str=""):
+	"""
+    Builds a TCP packet with the values specified by the caller.
+    
+    :param ip_src: the source IP address of the IP header
+    :param ip_dst the destination IP address of the IP header
+    :param mac_src: the source MAC address of the MAC header
+    :param ttl: the ttl value of the packet
+    :param mac_dst: the destination MAC address of the MAC header
+    :param port_src: the source port of the TCP header
+    :param port_dst: the destination port of the TCP header
+    :param tcpflags: the TCP flags of the TCP header
+    :param payload: the payload of the packet
+    :return: the corresponding TCP packet
+    """
+
+	ether = Ether(src=mac_src, dst=mac_dst)
+	ip = IP(src=ip_src, dst=ip_dst, ttl=ttl)
+	tcp = TCP(sport=port_src, dport=port_dst, flags=tcpflags)
+	packet = ether / ip / tcp / Raw(load=payload)
+	return packet
+
+
+def generate_udp_packet(ip_src:str="192.168.64.32", ip_dst:str="192.168.64.48", mac_src:str="56:6D:D9:BC:70:1C", ttl: int=64,
+					   mac_dst:str="F4:2B:95:B3:0E:1A", port_src:int=1337, port_dst:int=6442, payload:str=""):
+	"""
+    Builds an UDP packet with the values specified by the caller.
+    
+    :param ip_src: the source IP address of the IP header
+    :param ip_dst the destination IP address of the IP header
+    :param mac_src: the source MAC address of the MAC header
+    :param ttl: the ttl value of the packet
+    :param mac_dst: the destination MAC address of the MAC header
+    :param port_src: the source port of the UDP header
+    :param port_dst: the destination port of the UDP header
+    :param payload: the payload of the packet
+    :return: the corresponding UDP packet
+    """
+
+	ether = Ether(src=mac_src, dst=mac_dst)
+	ip = IP(src=ip_src, dst=ip_dst, ttl=ttl)
+	udp = UDP(sport=port_src, dport=port_dst)
+	packet = ether / ip / udp / Raw(load=payload)
+	return packet

code/ID2TLib/OldLibs/PaddingGenerator.py

@@ -0,0 +1,49 @@
+from scapy.packet import Raw
+
+import ID2TLib.PayloadGenerator as PayloadGenerator
+import numpy.random as random
+
+
+def add_padding(packet, bytes_padding = 0, user_padding=True, rnd = False):
+    '''
+    Adds padding to a packet with the given amount of bytes, but a maximum of 100 bytes.
+    :param packet: the packet that will be extended with the additional payload
+    :param bytes_padding: the amount of bytes that will be appended to the packet
+    :param user_padding: true, if the function add_padding is called from another class and the user
+    sets the padding manually
+    :param rnd: adds a random padding betwing 0 and bytes_adding, if true
+    :return: the initial packet, extended with the wanted amount of bytes of padding
+    '''
+
+    if(user_padding and bytes_padding > 100):
+        bytes_padding = 100
+
+    if (rnd is True):
+        r = int(round(bytes_padding / 4))                  #sets bytes_padding to any number between 0 and bytes_padding
+        bytes_padding = random.random_integers(0, r) * 4   #, that's dividable by 4
+    payload = PayloadGenerator.generate_payload(bytes_padding)
+    packet[Raw].load += Raw(load=payload).load
+    return packet
+
+def equal_length(list_of_packets, length = 0, padding = 0):
+    '''
+    Equals the length of a given set of packets on the given length. If the given length is smaller than the largest
+    packet, all the other packets are extended to the largest packet's length.
+    :param list_of_packets: The given set of packet.
+    :param length: The length each packet should have.
+    :return: The set of extended packets.
+    '''
+
+    largest_packet = length
+    for packet in list_of_packets:
+        packet_length = len(packet)
+        if(packet_length > largest_packet):
+            largest_packet = packet_length
+
+    for packet in list_of_packets:
+        bytes_padding = largest_packet - len(packet)
+        if(bytes_padding > 0):
+            add_padding(packet, bytes_padding, False, False)
+            add_padding(packet, padding, False, True)
+
+    return list_of_packets

code/ID2TLib/OldLibs/PayloadGenerator.py

@@ -0,0 +1,14 @@
+from numpy.random import bytes
+
+def generate_payload(size:int=0):
+
+	"""
+	Generates a payload of random bytes of the given amount
+
+	:param size: number of generated bytes
+    :return: the generated payload
+	"""
+
+	payload = bytes(size)
+
+	return payload

code/ID2TLib/OldLibs/PortGenerator.py

@@ -0,0 +1,14 @@
+import random
+import string
+
+def gen_random_server_port(offset: int=2199):
+    """
+    Generates a valid random first and last character for a bots hostname
+    and computes a port from these two characters.
+    The default offset is chosen from a Sality implementation in 2011
+    :param offest: default value, which is added to the two ASCII values
+    :return: sum of two ASCII characters and the default value
+    """
+    firstLetter = random.choice(string.ascii_letters);
+    lastLetter = random.choice(string.ascii_letters + string.digits);
+    return (offset + ord(firstLetter) * ord(lastLetter));

code/ID2TLib/OtherGroupLib/Controller.py

@@ -0,0 +1,248 @@
+import os
+import sys
+import readline
+
+from ID2TLib.AttackController import AttackController
+from ID2TLib.LabelManager import LabelManager
+from ID2TLib.PcapFile import PcapFile
+from ID2TLib.Statistics import Statistics
+
+
+class Controller:
+    def __init__(self, pcap_file_path: str, do_extra_tests: bool):
+        """
+        Creates a new Controller, acting as a central coordinator for the whole application.
+        :param pcap_file_path:
+        """
+        # Fields
+        self.pcap_src_path = pcap_file_path.strip()
+        self.pcap_dest_path = ''
+        self.written_pcaps = []
+        self.do_extra_tests = do_extra_tests
+
+        # Initialize class instances
+        print("Input file: %s" % self.pcap_src_path)
+        self.pcap_file = PcapFile(self.pcap_src_path)
+        self.label_manager = LabelManager(self.pcap_src_path)
+        self.statistics = Statistics(self.pcap_file)
+        self.statistics.do_extra_tests = self.do_extra_tests
+        self.statisticsDB = self.statistics.get_statistics_database()
+        self.attack_controller = AttackController(self.pcap_file, self.statistics, self.label_manager)
+
+    def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool):
+        """
+        Loads the PCAP statistics either from the database, if the statistics were calculated earlier, or calculates
+        the statistics and creates a new database.
+        :param flag_write_file: Writes the statistics to a file.
+        :param flag_recalculate_stats: Forces the recalculation of statistics.
+        :param flag_print_statistics: Prints the statistics on the terminal.
+        :return: None
+        """
+        self.statistics.load_pcap_statistics(flag_write_file, flag_recalculate_stats, flag_print_statistics)
+
+    def process_attacks(self, attacks_config: list):
+        """
+        Creates the attack based on the attack name and the attack parameters given in the attacks_config. The
+        attacks_config is a list of attacks, e.g.
+        [['PortscanAttack', 'ip.src="192.168.178.2",'dst.port=80'],['PortscanAttack', 'ip.src="10.10.10.2"]].
+        Merges the individual temporary attack pcaps into one single pcap and merges this single pcap with the
+        input dataset.
+        :param attacks_config: A list of attacks with their attack parameters.
+        """
+        # load attacks sequentially
+        for attack in attacks_config:
+            temp_attack_pcap = self.attack_controller.process_attack(attack[0], attack[1:])
+            self.written_pcaps.append(temp_attack_pcap)
+
+        # merge attack pcaps to get single attack pcap
+        if len(self.written_pcaps) > 1:
+            print("\nMerging temporary attack pcaps into single pcap file...", end=" ")
+            sys.stdout.flush()  # force python to print text immediately
+            for i in range(0, len(self.written_pcaps) - 1):
+                attacks_pcap = PcapFile(self.written_pcaps[i])
+                attacks_pcap_path = attacks_pcap.merge_attack(self.written_pcaps[i + 1])
+                os.remove(self.written_pcaps[i + 1])  # remove merged pcap
+                self.written_pcaps[i + 1] = attacks_pcap_path
+            print("done.")
+        else:
+            attacks_pcap_path = self.written_pcaps[0]
+
+        # merge single attack pcap with all attacks into base pcap
+        print("Merging base pcap with single attack pcap...", end=" ")
+        sys.stdout.flush()  # force python to print text immediately
+        self.pcap_dest_path = self.pcap_file.merge_attack(attacks_pcap_path)
+        print("done.")
+
+        # delete intermediate PCAP files
+        print('Deleting intermediate attack pcap...', end=" ")
+        sys.stdout.flush()  # force python to print text immediately
+        os.remove(attacks_pcap_path)
+        print("done.")
+
+        # write label file with attacks
+        self.label_manager.write_label_file(self.pcap_dest_path)
+
+        # print status message
+        print('\nOutput files created: \n', self.pcap_dest_path, '\n', self.label_manager.label_file_path)
+
+    def process_db_queries(self, query, print_results=False):
+        """
+        Processes a statistics database query. This can be a standard SQL query or a named query.
+        :param query: The query as a string or multiple queries as a list of strings.
+        :param print_results: Must be True if the results should be printed to terminal.
+        :return: The query's result
+        """
+        print("Processing database query/queries...")
+        if isinstance(query, list) or isinstance(query, tuple):
+            for q in query:
+                self.statisticsDB.process_db_query(q, print_results)
+        else:
+            self.statisticsDB.process_db_query(query, print_results)
+
+    @staticmethod
+    def process_help(params):
+        if not params:
+            print("Query mode allows you to enter SQL-queries as well as named queries.")
+            print()
+            print("Named queries:")
+            print("\tSelectors:")
+            print("\t\tmost_used(...)  -> Returns the most occurring element in all elements")
+            print("\t\tleast_used(...) -> Returns the least occurring element in all elements")
+            print("\t\tavg(...)        -> Returns the average of all elements")
+            print("\t\tall(...)        -> Returns all elements")
+            print("\tExtractors:")
+            print("\t\trandom(...)     -> Returns a random element from a list")
+            print("\t\tfirst(...)      -> Returns the first element from a list")
+            print("\t\tlast(...)       -> Returns the last element from a list")
+            print("\tParameterized selectors:")
+            print("\t\tipAddress(...)  -> Returns all IP addresses fulfilling the specified conditions")
+            print("\t\tmacAddress(...) -> Returns all MAC addresses fulfilling the specified conditions")
+            print()
+            print("Miscellaneous:")
+            print("\tlabels            -> List all attacks listed in the label file, if any")
+            print()
+            print("Additional information is available with 'help [KEYWORD];'")
+            print("To get a list of examples, type 'help examples;'")
+            print()
+            return
+
+        param = params[0].lower()
+        if param == "most_used":
+            print("most_used can be used as a selector for the following attributes:")
+            print("ipAddress | macAddress | portNumber | protocolName | ttlValue | mssValue | winSize | ipClass")
+            print()
+        elif param == "least_used":
+            print("least_used can be used as a selector for the following attributes:")
+            print("ipAddress | macAddress | portNumber | protocolName | ttlValue")
+            print()
+        elif param == "avg":
+            print("avg can be used as a selector for the following attributes:")
+            print("pktsReceived | pktsSent | kbytesSent | kbytesReceived | ttlValue | mss")
+            print()
+        elif param == "all":
+            print("all can be used as a selector for the following attributes:")
+            print("ipAddress | ttlValue | mss | macAddress | portNumber | protocolName")
+            print()
+        elif param in ["random", "first", "last"]:
+            print("No additional info available for this keyword.")
+            print()
+        elif param == "ipaddress":
+            print("ipAddress is a parameterized selector which fetches IP addresses based on (a list of) conditions.")
+            print("Conditions are of the following form: PARAMETER OPERATOR VALUE")
+            print("The following parameters can be specified:")
+            print("pktsReceived | pktsSent | kbytesReceived | kbytesSent | maxPktRate | minPktRate | ipClass\n"
+                  "macAddress | ttlValue | ttlCount | portDirection | portNumber | portCount | protocolCount\n"
+                  "protocolName")
+            print()
+            print("See 'help examples;' for usage examples.")
+            print()
+        elif param == "macaddress":
+            print("macAddress is a parameterized selector which fetches MAC addresses based on (a list of) conditions.")
+            print("Conditions are of the following form: PARAMETER OPERATOR VALUE")
+            print("The following parameters can be specified:")
+            print("ipAddress")
+            print()
+            print("See 'help examples;' for usage examples.")
+            print()
+        elif param == "examples":
+            print("Get the average amount of sent packets per IP:")
+            print("\tavg(pktsSent);")
+            print("Get a random IP from all addresses occuring in the pcap:")
+            print("\trandom(all(ipAddress));")
+            print("Return the MAC address of a specified IP:")
+            print("\tmacAddress(ipAddress=192.168.178.2);")
+            print("Get the average TTL-value with SQL:")
+            print("\tSELECT avg(ttlValue) from ip_ttl;")
+            print("Get a random IP address from all addresses that sent and received at least 10 packets:")
+            print("\trandom(ipAddress(pktsSent > 10, pktsReceived > 10));")
+            print()
+        else:
+            print("Unknown keyword '" + param + "', try 'help;' to get a list of allowed keywords'")
+            print()
+
+    def enter_query_mode(self):
+        """
+        Enters into the query mode. This is a read-eval-print-loop, where the user can input named queries or SQL
+        queries and the results are printed.
+        """
+
+        def make_completer(vocabulary):
+            def custom_template(text, state):
+                results = [x for x in vocabulary if x.startswith(text)] + [None]
+                return results[state]
+            return custom_template
+
+        readline.parse_and_bind('tab: complete')
+        readline.set_completer(make_completer(self.statisticsDB.get_all_named_query_keywords()+self.statisticsDB.get_all_sql_query_keywords()))
+        history_file = os.path.join(os.path.expanduser('~'), 'ID2T_data', 'query_history')
+        try:
+            readline.read_history_file(history_file)
+        except IOError:
+            pass
+        print("Entering into query mode...")
+        print("Enter statement ending by ';' and press ENTER to send query. Exit by sending an empty query.")
+        print("Type 'help;' for information on possible queries.")
+        buffer = ""
+        while True:
+            line = input("> ")
+            if line == "":
+                break
+            buffer += line
+            import sqlite3
+            if sqlite3.complete_statement(buffer):
+                try:
+                    buffer = buffer.strip()
+                    if buffer.lower().startswith('help'):
+                        buffer = buffer.strip(';')
+                        self.process_help(buffer.split(' ')[1:])
+                    elif buffer.lower().strip() == 'labels;':
+                        if not self.label_manager.labels:
+                            print("No labels found.")
+                        else:
+                            print("Attacks listed in the label file:")
+                            print()
+                            for label in self.label_manager.labels:
+                                print("Attack name:     " + str(label.attack_name))
+                                print("Attack note:     " + str(label.attack_note))
+                                print("Start timestamp: " + str(label.timestamp_start))
+                                print("End timestamp:   " + str(label.timestamp_end))
+                                print()
+                        print()
+                    else:
+                        self.statisticsDB.process_db_query(buffer, True)
+                except sqlite3.Error as e:
+                    print("An error occurred:", e.args[0])
+                buffer = ""
+
+        readline.set_history_length(1000)
+        readline.write_history_file(history_file)
+
+    def create_statistics_plot(self, params: str):
+        """
+        Plots the statistics to a file by using the given customization parameters.
+        """
+        if params is not None and params[0] is not None:
+            params_dict = dict([z.split("=") for z in params])
+            self.statistics.plot_statistics(format=params_dict['format'])
+        else:
+            self.statistics.plot_statistics()

code/ID2TLib/OtherGroupLib/Label.py

@@ -0,0 +1,32 @@
+from functools import total_ordering
+
+
+@total_ordering
+class Label:
+    def __init__(self, attack_name, timestamp_start, timestamp_end, attack_note=""):
+        """
+        Creates a new attack label
+
+        :param attack_name: The name of the associated attack
+        :param timestamp_start: The timestamp as unix time of the first attack packet
+        :param timestamp_end: The timestamp as unix time of the last attack packet
+        :param attack_note: A note associated to the attack (optional)
+        """
+        self.attack_name = attack_name
+        self.timestamp_start = timestamp_start
+        self.timestamp_end = timestamp_end
+        self.attack_note = attack_note
+
+    def __eq__(self, other):
+        return self.timestamp == other.timestamp
+
+    def __lt__(self, other):
+        return self.timestamp_start < other.timestamp_start
+
+    def __gt__(self, other):
+        return self.timestamp_start > other.timestamp_start
+
+    def __str__(self):
+        return ''.join(
+            ['(', self.attack_name, ',', self.attack_note, ',', str(self.timestamp_start), ',', str(self.timestamp_end),
+             ')'])

code/ID2TLib/OtherGroupLib/LabelManager.py

@@ -0,0 +1,169 @@
+import os.path
+from datetime import datetime
+from xml.dom.minidom import *
+
+import ID2TLib.Label as Label
+
+
+class LabelManager:
+    TAG_ROOT = 'LABELS'
+    TAG_ATTACK = 'attack'
+    TAG_ATTACK_NAME = 'attack_name'
+    TAG_ATTACK_NOTE = 'attack_note'
+    TAG_TIMESTAMP_START = 'timestamp_start'
+    TAG_TIMESTAMP_END = 'timestamp_end'
+    TAG_TIMESTAMP = 'timestamp'
+    TAG_TIMESTAMP_HR = 'timestamp_hr'
+    ATTR_VERSION = 'version_parser'
+
+    # update this attribute if XML scheme was modified
+    ATTR_VERSION_VALUE = '0.2'
+
+    def __init__(self, filepath_pcap=None):
+        """
+        Creates a new LabelManager for managing the attack's labels.
+
+        :param filepath_pcap: The path to the PCAP file associated to the labels.
+        """
+        self.labels = list()
+
+        if filepath_pcap is not None:
+            self.label_file_path = filepath_pcap.strip('.pcap') + '_labels.xml'
+            # only load labels if label file is existing
+            if os.path.exists(self.label_file_path):
+                self.load_labels()
+
+    def add_labels(self, labels):
+        """
+        Adds a label to the internal list of labels.
+
+        :param labels: The labels to be added
+        """
+        if isinstance(labels, list):
+            self.labels = self.labels + [labels]
+        elif isinstance(labels, tuple):
+            for l in labels:
+                self.labels.append(l)
+        else:
+            self.labels.append(labels)
+
+        # sorts the labels ascending by their timestamp
+        self.labels.sort()
+
+    def write_label_file(self, filepath=None):
+        """
+        Writes previously added/loaded labels to a XML file. Uses the given filepath as destination path, if no path is
+        given, uses the path in label_file_path.
+
+        :param filepath: The path where the label file should be written to.
+        """
+
+        def get_subtree_timestamp(xml_tag_root, timestamp_entry):
+            """
+            Creates the subtree for a given timestamp, consisting of the unix time format (seconds) and a human-readable
+            output.
+
+            :param xml_tag_root: The tag name for the root of the subtree
+            :param timestamp_entry: The timestamp as unix time
+            :return: The root node of the XML subtree
+            """
+            timestamp_root = doc.createElement(xml_tag_root)
+
+            # add timestamp in unix format
+            timestamp = doc.createElement(self.TAG_TIMESTAMP)
+            timestamp.appendChild(doc.createTextNode(str(timestamp_entry)))
+            timestamp_root.appendChild(timestamp)
+
+            # add timestamp in human-readable format
+            timestamp_hr = doc.createElement(self.TAG_TIMESTAMP_HR)
+            timestamp_hr_text = datetime.fromtimestamp(timestamp_entry).strftime('%Y-%m-%d %H:%M:%S.%f')
+            timestamp_hr.appendChild(doc.createTextNode(timestamp_hr_text))
+            timestamp_root.appendChild(timestamp_hr)
+
+            return timestamp_root
+
+        if filepath is not None:
+            self.label_file_path = filepath.strip('.pcap') + '_labels.xml'
+
+        # Generate XML
+        doc = Document()
+        node = doc.createElement(self.TAG_ROOT)
+        node.setAttribute(self.ATTR_VERSION, self.ATTR_VERSION_VALUE)
+        for label in self.labels:
+            xml_tree = doc.createElement(self.TAG_ATTACK)
+
+            # add attack to XML tree
+            attack_name = doc.createElement(self.TAG_ATTACK_NAME)
+            attack_name.appendChild(doc.createTextNode(str(label.attack_name)))
+            xml_tree.appendChild(attack_name)
+            attack_note = doc.createElement(self.TAG_ATTACK_NOTE)
+            attack_note.appendChild(doc.createTextNode(str(label.attack_note)))
+            xml_tree.appendChild(attack_note)
+
+            # add timestamp_start to XML tree
+            xml_tree.appendChild(get_subtree_timestamp(self.TAG_TIMESTAMP_START, label.timestamp_start))
+
+            # add timestamp_end to XML tree
+            xml_tree.appendChild(get_subtree_timestamp(self.TAG_TIMESTAMP_END, label.timestamp_end))
+
+            node.appendChild(xml_tree)
+
+        doc.appendChild(node)
+
+        # Write XML to file
+        file = open(self.label_file_path, 'w')
+        file.write(doc.toprettyxml())
+        file.close()
+
+    def load_labels(self):
+        """
+        Loads the labels from an already existing label XML file located at label_file_path (set by constructor).
+
+        """
+
+        def get_value_from_node(node, tag_name, *child_number):
+            """
+            Returns the value located in the tag specified by tag_name from a given node. Walks therefor the
+            node's children along as indicated by child_number, e.g., childNumber = (1,2,) first goes to the 1st child, and
+            then to the 2nd child of the first child -> elem.childNodes[1].childNodes[2].
+            """
+            elem = node.getElementsByTagName(tag_name)
+            if len(elem) == 1:
+                elem = elem[0]
+                for c in child_number:
+                    if len(elem.childNodes) > 0:
+                        elem = elem.childNodes[c]
+                    else:
+                        return ""
+                return elem.data
+            else:
+                return ""
+
+        print("Label file found. Loading labels...")
+        try:
+            dom = parse(self.label_file_path)
+        except Exception:
+            print('ERROR: Provided label file could not be parsed. Ignoring label file')
+            return
+
+        # Check if version of parser and version of file match
+        version = dom.getElementsByTagName(self.TAG_ROOT)
+        if len(version) > 0:
+            version = version[0].getAttribute(self.ATTR_VERSION)
+            if version == [] or not version == self.ATTR_VERSION_VALUE:
+                print(
+                    "The file " + self.label_file_path + " was created by another version of ID2TLib.LabelManager. Ignoring label file.")
+
+        # Parse attacks from XML file
+        attacks = dom.getElementsByTagName(self.TAG_ATTACK)
+        count_labels = 0
+        for a in attacks:
+            attack_name = get_value_from_node(a, self.TAG_ATTACK_NAME, 0)
+            attack_note = get_value_from_node(a, self.TAG_ATTACK_NOTE, 0)
+            timestamp_start = get_value_from_node(a, self.TAG_TIMESTAMP_START, 1, 0)
+            timestamp_end = get_value_from_node(a, self.TAG_TIMESTAMP_END, 1, 0)
+            label = Label.Label(attack_name, float(timestamp_start), float(timestamp_end), attack_note)
+            self.labels.append(label)
+            count_labels += 1
+
+        print("Read " + str(count_labels) + " label(s) successfully.")

code/ID2TLib/OtherGroupLib/Statistics.py

@@ -0,0 +1,963 @@
+from operator import itemgetter
+from math import sqrt, ceil, log
+
+import os
+import time
+import ID2TLib.libpcapreader as pr
+import matplotlib
+
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+from ID2TLib.PcapFile import PcapFile
+from ID2TLib.StatsDatabase import StatsDatabase
+
+
+class Statistics:
+    def __init__(self, pcap_file: PcapFile):
+        """
+        Creates a new Statistics object.
+
+        :param pcap_file: A reference to the PcapFile object
+        """
+        # Fields
+        self.pcap_filepath = pcap_file.pcap_file_path
+        self.pcap_proc = None
+        self.do_extra_tests = False
+
+        # Create folder for statistics database if required
+        self.path_db = pcap_file.get_db_path()
+        path_dir = os.path.dirname(self.path_db)
+        if not os.path.isdir(path_dir):
+            os.makedirs(path_dir)
+
+        # Class instances
+        self.stats_db = StatsDatabase(self.path_db)
+
+    def load_pcap_statistics(self, flag_write_file: bool, flag_recalculate_stats: bool, flag_print_statistics: bool):
+        """
+        Loads the PCAP statistics for the file specified by pcap_filepath. If the database is not existing yet, the
+        statistics are calculated by the PCAP file processor and saved into the newly created database. Otherwise the
+        statistics are gathered directly from the existing database.
+
+        :param flag_write_file: Indicates whether the statistics should be written addiotionally into a text file (True)
+        or not (False)
+        :param flag_recalculate_stats: Indicates whether eventually existing statistics should be recalculated
+        :param flag_print_statistics: Indicates whether the gathered basic statistics should be printed to the terminal
+        """
+        # Load pcap and get loading time
+        time_start = time.clock()
+
+        # Inform user about recalculation of statistics and its reason
+        if flag_recalculate_stats:
+            print("Flag -r/--recalculate found. Recalculating statistics.")
+
+        # Recalculate statistics if database does not exist OR param -r/--recalculate is provided
+        if (not self.stats_db.get_db_exists()) or flag_recalculate_stats:
+            self.pcap_proc = pr.pcap_processor(self.pcap_filepath, str(self.do_extra_tests))
+            self.pcap_proc.collect_statistics()
+            self.pcap_proc.write_to_database(self.path_db)
+            outstring_datasource = "by PCAP file processor."
+        else:
+            outstring_datasource = "from statistics database."
+
+        # Load statistics from database
+        self.file_info = self.stats_db.get_file_info()
+
+        time_end = time.clock()
+        print("Loaded file statistics in " + str(time_end - time_start)[:4] + " sec " + outstring_datasource)
+
+        # Write statistics if param -e/--export provided
+        if flag_write_file:
+            self.write_statistics_to_file()
+
+        # Print statistics if param -s/--statistics provided
+        if flag_print_statistics:
+            self.print_statistics()
+
+    def get_file_information(self):
+        """
+        Returns a list of tuples, each containing a information of the file.
+
+        :return: a list of tuples, each consisting of (description, value, unit), where unit is optional.
+        """
+        return [("Pcap file", self.pcap_filepath),
+                ("Packets", self.get_packet_count(), "packets"),
+                ("Capture length", self.get_capture_duration(), "seconds"),
+                ("Capture start", self.get_pcap_timestamp_start()),
+                ("Capture end", self.get_pcap_timestamp_end())]
+
+    def get_general_file_statistics(self):
+        """
+        Returns a list of tuples, each containing a file statistic.
+
+        :return: a list of tuples, each consisting of (description, value, unit).
+        """
+        return [("Avg. packet rate", self.file_info['avgPacketRate'], "packets/sec"),
+                ("Avg. packet size", self.file_info['avgPacketSize'], "kbytes"),
+                ("Avg. packets sent", self.file_info['avgPacketsSentPerHost'], "packets"),
+                ("Avg. bandwidth in", self.file_info['avgBandwidthIn'], "kbit/s"),
+                ("Avg. bandwidth out", self.file_info['avgBandwidthOut'], "kbit/s")]
+
+    @staticmethod
+    def write_list(desc_val_unit_list, func, line_ending="\n"):
+        """
+        Takes a list of tuples (statistic name, statistic value, unit) as input, generates a string of these three values
+        and applies the function func on this string.
+
+        Before generating the string, it identifies text containing a float number, casts the string to a
+        float and rounds the value to two decimal digits.
+
+        :param desc_val_unit_list: The list of tuples consisting of (description, value, unit)
+        :param func: The function to be applied to each generated string
+        :param line_ending: The formatting string to be applied at the end of each string
+        """
+        for entry in desc_val_unit_list:
+            # Convert text containing float into float
+            (description, value) = entry[0:2]
+            if isinstance(value, str) and "." in value:
+                try:
+                    value = float(value)
+                except ValueError:
+                    pass  # do nothing -> value was not a float
+            # round float
+            if isinstance(value, float):
+                value = round(value, 4)
+            # write into file
+            if len(entry) == 3:
+                unit = entry[2]
+                func(description + ":\t" + str(value) + " " + unit + line_ending)
+            else:
+                func(description + ":\t" + str(value) + line_ending)
+
+    def print_statistics(self):
+        """
+        Prints the basic file statistics to the terminal.
+        """
+        print("\nPCAP FILE INFORMATION ------------------------------")
+        Statistics.write_list(self.get_file_information(), print, "")
+        print("\nGENERAL FILE STATISTICS ----------------------------")
+        Statistics.write_list(self.get_general_file_statistics(), print, "")
+        print("\n")
+
+
+    def calculate_entropy(self, frequency:list, normalized:bool = False):
+        """
+        Calculates entropy and normalized entropy of list of elements that have specific frequency
+        :param frequency: The frequency of the elements.
+        :param normalized: Calculate normalized entropy
+        :return: entropy or (entropy, normalized entropy)
+        """
+        entropy, normalizedEnt, n = 0, 0, 0
+        sumFreq = sum(frequency)
+        for i, x in enumerate(frequency):
+            p_x = float(frequency[i] / sumFreq)
+            if p_x > 0:
+                n += 1
+                entropy += - p_x * log(p_x, 2)
+        if normalized:
+            if log(n)>0:
+                normalizedEnt = entropy/log(n, 2)
+            return entropy, normalizedEnt
+        else:
+            return entropy
+
+    def calculate_complement_packet_rates(self, pps):
+        """
+        Calculates the complement packet rates of the background traffic packet rates for each interval.
+        Then normalize it to maximum boundary, which is the input parameter pps
+
+        :return: normalized packet rates for each time interval.
+        """
+        result = self.process_db_query(
+            "SELECT lastPktTimestamp,pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
+        # print(result)
+        bg_interval_pps = []
+        complement_interval_pps = []
+        intervalsSum = 0
+        if result:
+            # Get the interval in seconds
+            for i, row in enumerate(result):
+                if i < len(result) - 1:
+                    intervalsSum += ceil((int(result[i + 1][0]) * 10 ** -6) - (int(row[0]) * 10 ** -6))
+            interval = intervalsSum / (len(result) - 1)
+            # Convert timestamp from micro to seconds, convert packet rate "per interval" to "per second"
+            for row in result:
+                bg_interval_pps.append((int(row[0]) * 10 ** -6, int(row[1] / interval)))
+            # Find max PPS
+            maxPPS = max(bg_interval_pps, key=itemgetter(1))[1]
+
+            for row in bg_interval_pps:
+                complement_interval_pps.append((row[0], int(pps * (maxPPS - row[1]) / maxPPS)))
+
+        return complement_interval_pps
+
+
+    def get_tests_statistics(self):
+        """
+        Writes the calculated basic defects tests statistics into a file.
+        """
+        # self.stats_db._process_user_defined_query output is list of tuples, thus, we ned [0][0] to access data
+
+        def count_frequncy(valuesList):
+            values, frequency = [] , []
+            for x in valuesList:
+                if x in values:
+                    frequency[values.index(x)] += 1
+                else:
+                    values.append(x)
+                    frequency.append(1)
+            return values, frequency
+
+        ####### Payload Tests #######
+        sumPayloadCount = self.stats_db._process_user_defined_query("SELECT sum(payloadCount) FROM interval_statistics")
+        pktCount = self.stats_db._process_user_defined_query("SELECT packetCount FROM file_statistics")
+        if sumPayloadCount and pktCount:
+            payloadRatio=0
+            if(pktCount[0][0]!=0):
+                payloadRatio = float(sumPayloadCount[0][0] / pktCount[0][0] * 100)
+        else:
+            payloadRatio = -1
+
+        ####### TCP checksum Tests #######
+        incorrectChecksumCount = self.stats_db._process_user_defined_query("SELECT sum(incorrectTCPChecksumCount) FROM interval_statistics")
+        correctChecksumCount = self.stats_db._process_user_defined_query("SELECT avg(correctTCPChecksumCount) FROM interval_statistics")
+        if incorrectChecksumCount and correctChecksumCount:
+            incorrectChecksumRatio=0
+            if(incorrectChecksumCount[0][0] + correctChecksumCount[0][0])!=0:
+                incorrectChecksumRatio = float(incorrectChecksumCount[0][0]  / (incorrectChecksumCount[0][0] + correctChecksumCount[0][0] ) * 100)
+        else:
+            incorrectChecksumRatio = -1
+
+        ####### IP Src & Dst Tests #######
+        result = self.stats_db._process_user_defined_query("SELECT ipAddress,pktsSent,pktsReceived FROM ip_statistics")
+        data, srcFrequency, dstFrequency = [], [], []
+        if result:
+            for row in result:
+                srcFrequency.append(row[1])
+                dstFrequency.append(row[2])
+        ipSrcEntropy, ipSrcNormEntropy = self.calculate_entropy(srcFrequency, True)
+        ipDstEntropy, ipDstNormEntropy = self.calculate_entropy(dstFrequency, True)
+
+        newIPCount = self.stats_db._process_user_defined_query("SELECT newIPCount FROM interval_statistics")
+        ipNovelsPerInterval, ipNovelsPerIntervalFrequency = count_frequncy(newIPCount)
+        ipNoveltyDistEntropy = self.calculate_entropy(ipNovelsPerIntervalFrequency)
+
+        ####### Ports Tests #######
+        port0Count = self.stats_db._process_user_defined_query("SELECT SUM(portCount) FROM ip_ports WHERE portNumber = 0")
+        if not port0Count[0][0]:
+            port0Count = 0
+        else:
+            port0Count = port0Count[0][0]
+        reservedPortCount = self.stats_db._process_user_defined_query(
+            "SELECT SUM(portCount) FROM ip_ports WHERE portNumber IN (100,114,1023,1024,49151,49152,65535)")# could be extended
+        if not reservedPortCount[0][0]:
+            reservedPortCount = 0
+        else:
+            reservedPortCount = reservedPortCount[0][0]
+
+        ####### TTL Tests #######
+        result = self.stats_db._process_user_defined_query("SELECT ttlValue,SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
+        data, frequency = [], []
+        for row in result:
+            frequency.append(row[1])
+        ttlEntropy, ttlNormEntropy  = self.calculate_entropy(frequency,True)
+        newTTLCount = self.stats_db._process_user_defined_query("SELECT newTTLCount FROM interval_statistics")
+        ttlNovelsPerInterval, ttlNovelsPerIntervalFrequency = count_frequncy(newTTLCount)
+        ttlNoveltyDistEntropy = self.calculate_entropy(ttlNovelsPerIntervalFrequency)
+
+        ####### Window Size Tests #######
+        result = self.stats_db._process_user_defined_query("SELECT winSize,SUM(winCount) FROM tcp_win GROUP BY winSize")
+        data, frequency = [], []
+        for row in result:
+            frequency.append(row[1])
+        winEntropy, winNormEntropy = self.calculate_entropy(frequency, True)
+        newWinSizeCount = self.stats_db._process_user_defined_query("SELECT newWinSizeCount FROM interval_statistics")
+        winNovelsPerInterval, winNovelsPerIntervalFrequency = count_frequncy(newWinSizeCount)
+        winNoveltyDistEntropy = self.calculate_entropy(winNovelsPerIntervalFrequency)
+
+        ####### ToS Tests #######
+        result = self.stats_db._process_user_defined_query(
+            "SELECT tosValue,SUM(tosCount) FROM ip_tos GROUP BY tosValue")
+        data, frequency = [], []
+        for row in result:
+            frequency.append(row[1])
+        tosEntropy, tosNormEntropy = self.calculate_entropy(frequency, True)
+        newToSCount = self.stats_db._process_user_defined_query("SELECT newToSCount FROM interval_statistics")
+        tosNovelsPerInterval, tosNovelsPerIntervalFrequency = count_frequncy(newToSCount)
+        tosNoveltyDistEntropy = self.calculate_entropy(tosNovelsPerIntervalFrequency)
+
+        ####### MSS Tests #######
+        result = self.stats_db._process_user_defined_query(
+            "SELECT mssValue,SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
+        data, frequency = [], []
+        for row in result:
+            frequency.append(row[1])
+        mssEntropy, mssNormEntropy = self.calculate_entropy(frequency, True)
+        newMSSCount = self.stats_db._process_user_defined_query("SELECT newMSSCount FROM interval_statistics")
+        mssNovelsPerInterval, mssNovelsPerIntervalFrequency = count_frequncy(newMSSCount)
+        mssNoveltyDistEntropy = self.calculate_entropy(mssNovelsPerIntervalFrequency)
+
+        result = self.stats_db._process_user_defined_query("SELECT SUM(mssCount) FROM tcp_mss WHERE mssValue > 1460")
+        # The most used MSS < 1460. Calculate the ratio of the values bigger that 1460.
+        if not result[0][0]:
+            result = 0
+        else:
+            result = result[0][0]
+        bigMSS = (result / sum(frequency)) * 100
+
+        output = []
+        if self.do_extra_tests:
+            output = [("Payload ratio", payloadRatio, "%"),
+                ("Incorrect TCP checksum ratio", incorrectChecksumRatio, "%")]
+
+        output = output + [("# IP addresses", sum([x[0] for x in newIPCount]), ""),
+                ("IP Src Entropy", ipSrcEntropy, ""),
+                ("IP Src Normalized Entropy", ipSrcNormEntropy, ""),
+                ("IP Dst Entropy", ipDstEntropy, ""),
+                ("IP Dst Normalized Entropy", ipDstNormEntropy, ""),
+                ("IP Novelty Distribution Entropy", ipNoveltyDistEntropy, ""),
+                ("# TTL values", sum([x[0] for x in newTTLCount]), ""),
+                ("TTL Entropy", ttlEntropy, ""),
+                ("TTL Normalized Entropy", ttlNormEntropy, ""),
+                ("TTL Novelty Distribution Entropy", ttlNoveltyDistEntropy, ""),
+                ("# WinSize values", sum([x[0] for x in newWinSizeCount]), ""),
+                ("WinSize Entropy", winEntropy, ""),
+                ("WinSize Normalized Entropy", winNormEntropy, ""),
+                ("WinSize Novelty Distribution Entropy", winNoveltyDistEntropy, ""),
+                ("# ToS values",  sum([x[0] for x in newToSCount]), ""),
+                ("ToS Entropy", tosEntropy, ""),
+                ("ToS Normalized Entropy", tosNormEntropy, ""),
+                ("ToS Novelty Distribution Entropy", tosNoveltyDistEntropy, ""),
+                ("# MSS values", sum([x[0] for x in newMSSCount]), ""),
+                ("MSS Entropy", mssEntropy, ""),
+                ("MSS Normalized Entropy", mssNormEntropy, ""),
+                ("MSS Novelty Distribution Entropy", mssNoveltyDistEntropy, ""),
+                ("======================","","")]
+
+        # Reasoning the statistics values
+        if self.do_extra_tests:
+            if payloadRatio > 80:
+                output.append(("WARNING: Too high payload ratio", payloadRatio, "%."))
+            if payloadRatio < 30:
+                output.append(("WARNING: Too low payload ratio", payloadRatio, "% (Injecting attacks that are carried out in the packet payloads is not recommmanded)."))
+
+            if incorrectChecksumRatio > 5:
+                output.append(("WARNING: High incorrect TCP checksum ratio",incorrectChecksumRatio,"%."))
+
+        if ipSrcNormEntropy > 0.65:
+            output.append(("WARNING: High IP source normalized entropy",ipSrcNormEntropy,"."))
+        if ipSrcNormEntropy < 0.2:
+            output.append(("WARNING: Low IP source normalized entropy", ipSrcNormEntropy, "."))
+        if ipDstNormEntropy > 0.65:
+            output.append(("WARNING: High IP destination normalized entropy", ipDstNormEntropy, "."))
+        if ipDstNormEntropy < 0.2:
+            output.append(("WARNING: Low IP destination normalized entropy", ipDstNormEntropy, "."))
+
+        if ttlNormEntropy > 0.65:
+            output.append(("WARNING: High TTL normalized entropy", ttlNormEntropy, "."))
+        if ttlNormEntropy < 0.2:
+            output.append(("WARNING: Low TTL normalized entropy", ttlNormEntropy, "."))
+        if ttlNoveltyDistEntropy < 1:
+            output.append(("WARNING: Too low TTL novelty distribution entropy", ttlNoveltyDistEntropy,
+                           "(The distribution of the novel TTL values is suspicious)."))
+
+        if winNormEntropy > 0.6:
+            output.append(("WARNING: High Window Size normalized entropy", winNormEntropy, "."))
+        if winNormEntropy < 0.1:
+            output.append(("WARNING: Low Window Size normalized entropy", winNormEntropy, "."))
+        if winNoveltyDistEntropy < 4:
+            output.append(("WARNING: Low Window Size novelty distribution entropy", winNoveltyDistEntropy,
+                           "(The distribution of the novel Window Size values is suspicious)."))
+
+        if tosNormEntropy > 0.4:
+            output.append(("WARNING: High ToS normalized entropy", tosNormEntropy, "."))
+        if tosNormEntropy < 0.1:
+            output.append(("WARNING: Low ToS normalized entropy", tosNormEntropy, "."))
+        if tosNoveltyDistEntropy < 0.5:
+            output.append(("WARNING: Low ToS novelty distribution entropy", tosNoveltyDistEntropy,
+                           "(The distribution of the novel ToS values is suspicious)."))
+
+        if mssNormEntropy > 0.4:
+            output.append(("WARNING: High MSS normalized entropy", mssNormEntropy, "."))
+        if mssNormEntropy < 0.1:
+            output.append(("WARNING: Low MSS normalized entropy", mssNormEntropy, "."))
+        if mssNoveltyDistEntropy < 0.5:
+            output.append(("WARNING: Low MSS novelty distribution entropy", mssNoveltyDistEntropy,
+                           "(The distribution of the novel MSS values is suspicious)."))
+
+        if bigMSS > 50:
+            output.append(("WARNING: High ratio of MSS > 1460", bigMSS, "% (High fragmentation rate in Ethernet)."))
+
+        if port0Count > 0:
+            output.append(("WARNING: Port number 0 is used in ",port0Count,"packets (awkward-looking port)."))
+        if reservedPortCount > 0:
+            output.append(("WARNING: Reserved port numbers are used in ",reservedPortCount,"packets (uncommonly-used ports)."))
+
+        return output
+
+    def write_statistics_to_file(self):
+        """
+        Writes the calculated basic statistics into a file.
+        """
+
+        def _write_header(title: str):
+            """
+            Writes the section header into the open file.
+
+            :param title: The section title
+            """
+            target.write("====================== \n")
+            target.write(title + " \n")
+            target.write("====================== \n")
+
+        target = open(self.pcap_filepath + ".stat", 'w')
+        target.truncate()
+
+        _write_header("PCAP file information")
+        Statistics.write_list(self.get_file_information(), target.write)
+
+        _write_header("General statistics")
+        Statistics.write_list(self.get_general_file_statistics(), target.write)
+
+        _write_header("Tests statistics")
+        Statistics.write_list(self.get_tests_statistics(), target.write)
+
+        target.close()
+
+    def get_capture_duration(self):
+        """
+        :return: The duration of the capture in seconds
+        """
+        return self.file_info['captureDuration']
+
+    def get_pcap_timestamp_start(self):
+        """
+        :return: The timestamp of the first packet in the PCAP file
+        """
+        return self.file_info['timestampFirstPacket']
+
+    def get_pcap_timestamp_end(self):
+        """
+        :return: The timestamp of the last packet in the PCAP file
+        """
+        return self.file_info['timestampLastPacket']
+
+    def get_pps_sent(self, ip_address: str):
+        """
+        Calculates the sent packets per seconds for a given IP address.
+
+        :param ip_address: The IP address whose packets per second should be calculated
+        :return: The sent packets per seconds for the given IP address
+        """
+        packets_sent = self.stats_db.process_db_query("SELECT pktsSent from ip_statistics WHERE ipAddress=?", False,
+                                                      (ip_address,))
+        capture_duration = float(self.get_capture_duration())
+        return int(float(packets_sent) / capture_duration)
+
+    def get_pps_received(self, ip_address: str):
+        """
+        Calculate the packets per second received for a given IP address.
+
+        :param ip_address: The IP address used for the calculation
+        :return: The number of packets per second received
+        """
+        packets_received = self.stats_db.process_db_query("SELECT pktsReceived FROM ip_statistics WHERE ipAddress=?",
+                                                          False,
+                                                          (ip_address,))
+        capture_duration = float(self.get_capture_duration())
+        return int(float(packets_received) / capture_duration)
+
+    def get_packet_count(self):
+        """
+        :return: The number of packets in the loaded PCAP file
+        """
+        return self.file_info['packetCount']
+
+    def get_most_used_ip_address(self):
+        """
+        :return: The IP address/addresses with the highest sum of packets sent and received
+        """
+        return self.process_db_query("most_used(ipAddress)")
+
+    def get_ttl_distribution(self, ipAddress: str):
+        result = self.process_db_query('SELECT ttlValue, ttlCount from ip_ttl WHERE ipAddress="' + ipAddress + '"')
+        result_dict = {key: value for (key, value) in result}
+        return result_dict
+
+    def get_mss_distribution(self, ipAddress: str):
+        result = self.process_db_query('SELECT mssValue, mssCount from tcp_mss WHERE ipAddress="' + ipAddress + '"')
+        result_dict = {key: value for (key, value) in result}
+        return result_dict
+
+    def get_win_distribution(self, ipAddress: str):
+        result = self.process_db_query('SELECT winSize, winCount from tcp_win WHERE ipAddress="' + ipAddress + '"')
+        result_dict = {key: value for (key, value) in result}
+        return result_dict
+
+    def get_tos_distribution(self, ipAddress: str):
+        result = self.process_db_query('SELECT tosValue, tosCount from ip_tos WHERE ipAddress="' + ipAddress + '"')
+        result_dict = {key: value for (key, value) in result}
+        return result_dict
+
+    def get_ip_address_count(self):
+        return self.process_db_query("SELECT COUNT(*) FROM ip_statistics")
+
+    def get_ip_addresses(self):
+        return self.process_db_query("SELECT ipAddress FROM ip_statistics")
+
+    def get_random_ip_address(self, count: int = 1):
+        """
+        :param count: The number of IP addreses to return
+        :return: A randomly chosen IP address from the dataset or iff param count is greater than one, a list of randomly
+         chosen IP addresses
+        """
+        if count == 1:
+            return self.process_db_query("random(all(ipAddress))")
+        else:
+            ip_address_list = []
+            for i in range(0, count):
+                ip_address_list.append(self.process_db_query("random(all(ipAddress))"))
+            return ip_address_list
+
+    def get_ip_address_from_mac(self, macAddress: str):
+        """
+        :param macAddress: the MAC address of which the IP shall be returned, if existing in DB
+        :return: the IP address used in the dataset by a given MAC address
+        """
+        return self.process_db_query('ipAddress(macAddress=' + macAddress + ")")
+
+    def get_mac_address(self, ipAddress: str):
+        """
+        :return: The MAC address used in the dataset for the given IP address.
+        """
+        return self.process_db_query('macAddress(ipAddress=' + ipAddress + ")")
+
+    def get_most_used_mss(self, ipAddress: str):
+        """
+        :param ipAddress: The IP address whose used MSS should be determined
+        :return: The TCP MSS value used by the IP address, or if the IP addresses never specified a MSS,
+        then None is returned
+        """
+        mss_value = self.process_db_query('SELECT mssValue from tcp_mss WHERE ipAddress="' + ipAddress + '" ORDER BY mssCount DESC LIMIT 1')
+        if isinstance(mss_value, int):
+            return mss_value
+        else:
+            return None
+
+    def get_most_used_ttl(self, ipAddress: str):
+        """
+        :param ipAddress: The IP address whose used TTL should be determined
+        :return: The TTL value used by the IP address, or if the IP addresses never specified a TTL,
+        then None is returned
+        """
+        ttl_value = self.process_db_query(
+            'SELECT ttlValue from ip_ttl WHERE ipAddress="' + ipAddress + '" ORDER BY ttlCount DESC LIMIT 1')
+        if isinstance(ttl_value, int):
+            return ttl_value
+        else:
+            return None
+
+
+    def get_statistics_database(self):
+        """
+        :return: A reference to the statistics database object
+        """
+        return self.stats_db
+
+    def process_db_query(self, query_string_in: str, print_results: bool = False):
+        """
+        Executes a string identified previously as a query. This can be a standard SQL SELECT/INSERT query or a named
+        query.
+
+        :param query_string_in: The query to be processed
+        :param print_results: Indicates whether the results should be printed to terminal
+        :return: The result of the query
+        """
+        return self.stats_db.process_db_query(query_string_in, print_results)
+
+    def is_query(self, value: str):
+        """
+        Checks whether the given string is a standard SQL query (SELECT, INSERT) or a named query.
+
+        :param value: The string to be checked
+        :return: True if the string is recognized as a query, otherwise False.
+        """
+        if not isinstance(value, str):
+            return False
+        else:
+            return (any(x in value.lower().strip() for x in self.stats_db.get_all_named_query_keywords()) or
+                    any(x in value.lower().strip() for x in self.stats_db.get_all_sql_query_keywords()))
+
+
+    def calculate_standard_deviation(self, lst):
+        """
+        Calculates the standard deviation of a list of numbers.
+        :param lst: The list of numbers to calculate its SD.
+
+        """
+        num_items = len(lst)
+        mean = sum(lst) / num_items
+        differences = [x - mean for x in lst]
+        sq_differences = [d ** 2 for d in differences]
+        ssd = sum(sq_differences)
+        variance = ssd / num_items
+        sd = sqrt(variance)
+        return sd
+
+
+    def plot_statistics(self, format: str = 'pdf'): #'png'
+        """
+        Plots the statistics associated with the dataset.
+        :param format: The format to be used to save the statistics diagrams.
+        """
+
+        def plot_distribution(queryOutput, title,  xLabel, yLabel, file_ending: str):
+            plt.gcf().clear()
+            graphx, graphy = [], []
+            for row in queryOutput:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title(title)
+            plt.xlabel(xLabel)
+            plt.ylabel(yLabel)
+            width = 0.1
+            plt.xlim([0, max(graphx)])
+            plt.grid(True)
+            plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+        def plot_ttl(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT ttlValue, SUM(ttlCount) FROM ip_ttl GROUP BY ttlValue")
+            title = "TTL Distribution"
+            xLabel = "TTL Value"
+            yLabel = "Number of Packets"
+            if queryOutput:
+                return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_mss(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT mssValue, SUM(mssCount) FROM tcp_mss GROUP BY mssValue")
+            title = "MSS Distribution"
+            xLabel = "MSS Value"
+            yLabel = "Number of Packets"
+            if queryOutput:
+                return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_win(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT winSize, SUM(winCount) FROM tcp_win GROUP BY winSize")
+            title = "Window Size Distribution"
+            xLabel = "Window Size"
+            yLabel = "Number of Packets"
+            if queryOutput:
+                return plot_distribution(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_protocol(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT protocolName, SUM(protocolCount) FROM ip_protocols GROUP BY protocolName")
+            if (result):
+                graphx, graphy = [], []
+                for row in result:
+                    graphx.append(row[0])
+                    graphy.append(row[1])
+                plt.autoscale(enable=True, axis='both')
+                plt.title("Protocols Distribution")
+                plt.xlabel('Protocols')
+                plt.ylabel('Number of Packets')
+                width = 0.1
+                plt.xlim([0, len(graphx)])
+                plt.grid(True)
+
+                # Protocols' names on x-axis
+                x = range(0,len(graphx))
+                my_xticks = graphx
+                plt.xticks(x, my_xticks)
+
+                plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+                out = self.pcap_filepath.replace('.pcap', '_plot-protocol' + file_ending)
+                plt.savefig(out,dpi=500)
+                return out
+            else:
+                print("Error plot protocol: No protocol values found!")
+
+        def plot_port(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT portNumber, SUM(portCount) FROM ip_ports GROUP BY portNumber")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Ports Distribution")
+            plt.xlabel('Ports Numbers')
+            plt.ylabel('Number of Packets')
+            width = 0.1
+            plt.xlim([0, max(graphx)])
+            plt.grid(True)
+            plt.bar(graphx, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-port' + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+        # This distribution is not drawable for big datasets
+        def plot_ip_src(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT ipAddress, pktsSent FROM ip_statistics")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Source IP Distribution")
+            plt.xlabel('Source IP')
+            plt.ylabel('Number of Packets')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            # IPs on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-ip-src' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        # This distribution is not drawable for big datasets
+        def plot_ip_dst(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT ipAddress, pktsReceived FROM ip_statistics")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Destination IP Distribution")
+            plt.xlabel('Destination IP')
+            plt.ylabel('Number of Packets')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            # IPs on x-axis
+            x = range(0, len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+            plt.tight_layout()
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-ip-dst' + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        def plot_interval_statistics(queryOutput, title,  xLabel, yLabel, file_ending: str):
+            plt.gcf().clear()
+            graphx, graphy = [], []
+            for row in queryOutput:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title(title)
+            plt.xlabel(xLabel)
+            plt.ylabel(yLabel)
+            width = 0.5
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            # timestamp on x-axis
+            x = range(0, len(graphx))
+
+            # limit the number of xticks
+            plt.locator_params(axis='x', nbins=20)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_plot-' + title + file_ending)
+            plt.savefig(out, dpi=500)
+            return out
+
+        def plot_interval_pktCount(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, pktsCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "Packet Rate"
+            xLabel = "Time Interval"
+            yLabel = "Number of Packets"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_ip_src_ent(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, ipSrcEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "Source IP Entropy"
+            xLabel = "Time Interval"
+            yLabel = "Entropy"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_ip_dst_ent(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, ipDstEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "Destination IP Entropy"
+            xLabel = "Time Interval"
+            yLabel = "Entropy"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_ip(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newIPCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "IP Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_port(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newPortCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "Port Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_ttl(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newTTLCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "TTL Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_tos(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newToSCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "ToS Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_win_size(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newWinSizeCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "Window Size Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_new_mss(file_ending: str):
+            queryOutput = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, newMSSCount FROM interval_statistics ORDER BY lastPktTimestamp")
+            title = "MSS Novelty Distribution"
+            xLabel = "Time Interval"
+            yLabel = "Novel values count"
+            if queryOutput:
+                return plot_interval_statistics(queryOutput, title, xLabel, yLabel, file_ending)
+
+        def plot_interval_ip_dst_cum_ent(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, ipDstCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            # If entropy was not calculated do not plot the graph
+            if graphy[0] != -1:
+                plt.autoscale(enable=True, axis='both')
+                plt.title("Destination IP Cumulative Entropy")
+                # plt.xlabel('Timestamp')
+                plt.xlabel('Time Interval')
+                plt.ylabel('Entropy')
+                plt.xlim([0, len(graphx)])
+                plt.grid(True)
+
+                # timestamp on x-axis
+                x = range(0, len(graphx))
+                # my_xticks = graphx
+                # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+                # plt.tight_layout()
+
+                # limit the number of xticks
+                plt.locator_params(axis='x', nbins=20)
+
+                plt.plot(x, graphy, 'r')
+                out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-dst-cum-ent' + file_ending)
+                plt.savefig(out, dpi=500)
+                return out
+
+        def plot_interval_ip_src_cum_ent(file_ending: str):
+            plt.gcf().clear()
+
+            result = self.stats_db._process_user_defined_query(
+                "SELECT lastPktTimestamp, ipSrcCumEntropy FROM interval_statistics ORDER BY lastPktTimestamp")
+            graphx, graphy = [], []
+            for row in result:
+                graphx.append(row[0])
+                graphy.append(row[1])
+            # If entropy was not calculated do not plot the graph
+            if graphy[0] != -1:
+                plt.autoscale(enable=True, axis='both')
+                plt.title("Source IP Cumulative Entropy")
+                # plt.xlabel('Timestamp')
+                plt.xlabel('Time Interval')
+                plt.ylabel('Entropy')
+                plt.xlim([0, len(graphx)])
+                plt.grid(True)
+
+                # timestamp on x-axis
+                x = range(0, len(graphx))
+                # my_xticks = graphx
+                # plt.xticks(x, my_xticks, rotation='vertical', fontsize=5)
+                # plt.tight_layout()
+
+                # limit the number of xticks
+                plt.locator_params(axis='x', nbins=20)
+
+                plt.plot(x, graphy, 'r')
+                out = self.pcap_filepath.replace('.pcap', '_plot-interval-ip-src-cum-ent' + file_ending)
+                plt.savefig(out, dpi=500)
+                return out
+
+        ttl_out_path = plot_ttl('.' + format)
+        mss_out_path = plot_mss('.' + format)
+        win_out_path = plot_win('.' + format)
+        protocol_out_path = plot_protocol('.' + format)
+        plot_interval_pktCount = plot_interval_pktCount('.' + format)
+        plot_interval_ip_src_ent = plot_interval_ip_src_ent('.' + format)
+        plot_interval_ip_dst_ent = plot_interval_ip_dst_ent('.' + format)
+        plot_interval_ip_src_cum_ent = plot_interval_ip_src_cum_ent('.' + format)
+        plot_interval_ip_dst_cum_ent = plot_interval_ip_dst_cum_ent('.' + format)
+        plot_interval_new_ip = plot_interval_new_ip('.' + format)
+        plot_interval_new_port = plot_interval_new_port('.' + format)
+        plot_interval_new_ttl = plot_interval_new_ttl('.' + format)
+        plot_interval_new_tos = plot_interval_new_tos('.' + format)
+        plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
+        plot_interval_new_mss = plot_interval_new_mss('.' + format)
+
+        ## Time consuming plot
+        # port_out_path = plot_port('.' + format)
+        ## Not drawable for too many IPs
+        # ip_src_out_path = plot_ip_src('.' + format)
+        # ip_dst_out_path = plot_ip_dst('.' + format)
+
+        print("Saved plots in the input PCAP directory.")

code/ID2TLib/Statistics.py

@@ -10,6 +10,7 @@ matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 from ID2TLib.PcapFile import PcapFile
 from ID2TLib.StatsDatabase import StatsDatabase
+from ID2TLib.IPv4 import IPAddress
 
 
 class Statistics:
@@ -499,12 +500,6 @@ class Statistics:
         result_dict = {key: value for (key, value) in result}
         return result_dict
 
-    def get_ip_address_count(self):
-        return self.process_db_query("SELECT COUNT(*) FROM ip_statistics")
-
-    def get_ip_addresses(self):
-        return self.process_db_query("SELECT ipAddress FROM ip_statistics")
-
     def get_random_ip_address(self, count: int = 1):
         """
         :param count: The number of IP addreses to return
@@ -519,13 +514,6 @@ class Statistics:
                 ip_address_list.append(self.process_db_query("random(all(ipAddress))"))
             return ip_address_list
 
-    def get_ip_address_from_mac(self, macAddress: str):
-        """
-        :param macAddress: the MAC address of which the IP shall be returned, if existing in DB
-        :return: the IP address used in the dataset by a given MAC address
-        """
-        return self.process_db_query('ipAddress(macAddress=' + macAddress + ")")
-
     def get_mac_address(self, ipAddress: str):
         """
         :return: The MAC address used in the dataset for the given IP address.
@@ -557,6 +545,154 @@ class Statistics:
         else:
             return None
 
+    def get_in_degree(self):
+        """
+        determines the in-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has received packets from
+        :return: a list, each entry consists of one local IPAddress and its associated in-degree
+        """
+
+        in_degree_raw = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'in\' AND portNumber = portA GROUP BY ipAddress " +
+                "UNION " +
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'in\' AND portNumber = portB GROUP BY ipAddress")
+
+        #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
+        in_degree = self.filter_multiples(in_degree_raw)
+
+        return in_degree
+
+    def get_out_degree(self):
+        """
+        determines the out-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
+        :return: a list, each entry consists of one local IPAddress and its associated out-degree
+        """
+        """
+
+        test = self.stats_db._process_user_defined_query("SELECT DISTINCT * FROM conv_statistics")
+        #test2 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddressB, portB FROM conv_statistics")
+        print("############# conv_statistics IP's + Ports")
+        for p in test:
+            print(p)
+        #for p in test2:
+        #    print(p)
+
+        print("############## ip_ports ##################")
+        test3 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddress, portNumber, portDirection FROM ip_ports")
+        for p in test3:
+            print(p)
+
+        print("")
+        print("############## AFTER JOIN - A #############")
+        test4 = self.stats_db._process_user_defined_query(
+                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA") # Hier werden die anfang locals rausgefiltert!
+        for p in test4:
+            print(p)
+
+        print("")
+        print("############## AFTER JOIN - B #############")
+        test6 = self.stats_db._process_user_defined_query(
+                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB") # Hier werden die anfang locals rausgefiltert!
+        for p in test6:
+            print(p)
+
+        print("")
+        print("############## BUILD UP PART FOR PART#############")
+        test5 = self.stats_db._process_user_defined_query(
+                "SELECT ipAddress, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' GROUP BY ipAddress")
+        for p in test5:
+            print(p)
+        """
+        out_degree_raw = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA GROUP BY ipAddress " +
+                "UNION " +
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB GROUP BY ipAddress")
+
+        #filter out non-local IPs
+        #out_degree_raw_2 = []
+        #for entry in out_degree_raw:
+        #    if IPAddress.parse(entry[0]).is_reserved():
+        #        out_degree_raw_2.append(entry)
+
+        #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
+        out_degree = self.filter_multiples(out_degree_raw)
+
+        return out_degree
+
+    def get_avg_delay_local_ext(self):
+        """
+        Calculates the average delay of a packet for external and local communication, based on the tcp handshakes
+        :return: tuple consisting of avg delay for local and external communication, (local, external)
+        """
+
+        conv_delays = self.stats_db._process_user_defined_query("SELECT ipAddressA, ipAddressB, avgDelay FROM conv_statistics")
+        if(conv_delays):
+            external_conv = []
+            local_conv = []
+
+            for conv in conv_delays:
+                IPA = IPAddress.parse(conv[0])
+                IPB = IPAddress.parse(conv[1])
+
+                #split into local and external conversations
+                if(not IPA.is_private() or not IPB.is_private()):
+                    external_conv.append(conv)
+                else:
+                    local_conv.append(conv)
+   
+            # calculate avg local and external delay by summing up the respective delays and dividing them by the number of conversations
+            avg_delay_external = 0.0
+            avg_delay_local = 0.0
+
+            if(local_conv):
+                for conv in local_conv:
+                    avg_delay_local += conv[2]
+                avg_delay_local = (avg_delay_local/len(local_conv)) * 0.001 #ms
+            else:
+                # no local conversations in statistics found
+                avg_delay_local = 0.06
+
+            if(external_conv):
+                for conv in external_conv:
+                    avg_delay_external += conv[2]
+                avg_delay_external = (avg_delay_external/len(external_conv)) * 0.001 #ms
+            else:
+                # no external conversations in statistics found
+                avg_delay_external = 0.15
+        else:
+            #if no statistics were found, use these numbers
+            avg_delay_external = 0.15
+            avg_delay_local = 0.06
+        return avg_delay_local, avg_delay_external
+
+    def filter_multiples(self, entries):
+        """
+        helper function, for get_out_degree and get_in_degree
+        filters the given list for duplicate IpAddresses and, if duplciates are present, accumulates their values
+
+        :param entries: list, each entry consists of an ipAddress and a numeric value
+        :return: a filtered list, without duplicate ipAddresses
+        """
+
+        filtered_entries = []
+        done = []
+        for p1 in entries:
+            added = False
+            if p1 in done:
+                continue
+            for p2 in entries:
+                if p1[0] == p2[0] and p1 != p2:
+                    filtered_entries.append((p1[0], p1[1] + p2[1]))
+                    done.append(p1)
+                    done.append(p2)
+                    #entries.remove(p2)
+                    added = True
+                    break
+
+            if not added:
+                filtered_entries.append(p1)
+
+        return filtered_entries
+
 
     def get_statistics_database(self):
         """
@@ -938,6 +1074,190 @@ class Statistics:
                 plt.savefig(out, dpi=500)
                 return out
 
+        def plot_packets_per_connection(file_ending: str):
+            """
+            Plots the exchanged packets per connection as horizontal bar plot. 
+            Included are 'half-open' connections, where only one packet is exchanged.
+            Note: there may be cutoff problems within the plot if there is to little data.
+
+            :param file_ending: The file extension for the output file containing the plot
+            :return: A filepath to the file containing the created plot
+            """
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, portA, ipAddressB, portB, pktsCount FROM conv_statistics_stateless")
+
+            if (result):
+                graphy, graphx = [], []
+                # plot data in descending order
+                result = sorted(result, key=lambda row: row[4])
+
+                # compute plot data
+                for i, row in enumerate(result):
+                    addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
+                    # adjust the justification of strings to improve appearance
+                    len_max = max(len(addr1), len(addr2))
+                    addr1 = addr1.ljust(len_max)
+                    addr2 = addr2.ljust(len_max)
+                    # add plot data
+                    graphy.append("%s\n%s" % (addr1, addr2))
+                    graphx.append(row[4])
+
+                # compute plot height in inches
+                dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
+                plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
+                title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
+
+                # have x axis and its label appear at the top (instead of bottom)
+                fig, ax = plt.subplots()
+                ax.xaxis.tick_top()
+                ax.xaxis.set_label_position("top")
+
+                # set additional plot parameters
+                plt.title("Sent packets per connection", y=title_distance)
+                plt.xlabel('Number of Packets')
+                plt.ylabel('Connection')
+                width = 0.5
+                plt.grid(True)
+                plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot
+                plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
+
+                # plot the above data, first use plain numbers as graphy to maintain sorting
+                plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+                # now change the y numbers to the respective address labels
+                plt.yticks(range(len(graphy)), graphy)
+                # try to use tight layout to cut off unnecessary space
+                try:
+                    plt.tight_layout(pad=4)
+                except ValueError:
+                    pass
+
+                # save created figure
+                out = self.pcap_filepath.replace('.pcap', '_plot-PktCount per Connection Distribution' + file_ending)
+                plt.savefig(out, dpi=500)
+                return out
+            else:
+                print("Error plot protocol: No protocol values found!")
+
+        def plot_out_degree(file_ending: str):
+            plt.gcf().clear()
+            out_degree = self.get_out_degree()
+            #print("")
+            #print("#############in plot_out_degree###########")
+            #print(out_degree)
+
+            graphx, graphy = [], []
+            for entry in out_degree:
+                graphx.append(entry[0])
+                graphy.append(entry[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Outdegree")
+            plt.xlabel('IpAddress')
+            plt.ylabel('Outdegree')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            x = range(0,len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_out_degree' + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+        def plot_in_degree(file_ending: str):
+            plt.gcf().clear()
+            in_degree = self.get_in_degree()
+
+            graphx, graphy = [], []
+            for entry in in_degree:
+                graphx.append(entry[0])
+                graphy.append(entry[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Indegree")
+            plt.xlabel('IpAddress')
+            plt.ylabel('Indegree')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            x = range(0,len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_in_degree' + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+        def plot_avgpkts_per_comm_interval(file_ending: str):
+            """
+            Plots the exchanged packets per connection as horizontal bar plot. 
+            Included are 'half-open' connections, where only one packet is exchanged.
+            Note: there may be cutoff problems within the plot if there is to little data.
+
+            :param file_ending: The file extension for the output file containing the plot
+            :return: A filepath to the file containing the created plot
+            """
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, portA, ipAddressB, portB, avgPktCount FROM comm_interval_statistics")
+
+            if (result):
+                graphy, graphx = [], []
+                # plot data in descending order
+                result = sorted(result, key=lambda row: row[4])
+
+                # compute plot data
+                for i, row in enumerate(result):
+                    addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
+                    # adjust the justification of strings to improve appearance
+                    len_max = max(len(addr1), len(addr2))
+                    addr1 = addr1.ljust(len_max)
+                    addr2 = addr2.ljust(len_max)
+                    # add plot data
+                    graphy.append("%s\n%s" % (addr1, addr2))
+                    graphx.append(row[4])
+
+                # compute plot height in inches
+                dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
+                plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
+                title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
+
+                # have x axis and its label appear at the top (instead of bottom)
+                fig, ax = plt.subplots()
+                ax.xaxis.tick_top()
+                ax.xaxis.set_label_position("top")
+
+                # set additional plot parameters
+                plt.title("Average number of packets per communication interval", y=title_distance)
+                plt.xlabel('Number of Packets')
+                plt.ylabel('Connection')
+                width = 0.5
+                plt.grid(True)
+                plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot
+                plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
+
+                # plot the above data, first use plain numbers as graphy to maintain sorting
+                plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+                # now change the y numbers to the respective address labels
+                plt.yticks(range(len(graphy)), graphy)
+                # try to use tight layout to cut off unnecessary space
+                try:
+                    plt.tight_layout(pad=4)
+                except ValueError:
+                    pass
+
+                # save created figure
+                out = self.pcap_filepath.replace('.pcap', '_plot-Avg PktCount Communication Interval Distribution' + file_ending)
+                plt.savefig(out, dpi=500)
+                return out
+            else:
+                print("Error plot protocol: No protocol values found!")
+
+
         ttl_out_path = plot_ttl('.' + format)
         mss_out_path = plot_mss('.' + format)
         win_out_path = plot_win('.' + format)
@@ -953,6 +1273,10 @@ class Statistics:
         plot_interval_new_tos = plot_interval_new_tos('.' + format)
         plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
         plot_interval_new_mss = plot_interval_new_mss('.' + format)
+        plot_packets_per_connection_out = plot_packets_per_connection('.' + format)
+        plot_out_degree = plot_out_degree('.' + format)
+        plot_in_degree = plot_in_degree('.' + format)
+        plot_avgpkts_per_comm_interval_out = plot_avgpkts_per_comm_interval('.' + format)
 
         ## Time consuming plot
         # port_out_path = plot_port('.' + format)
@@ -961,3 +1285,4 @@ class Statistics:
         # ip_dst_out_path = plot_ip_dst('.' + format)
 
         print("Saved plots in the input PCAP directory.")
+        print("In-/Out-/Overall-degree plots not fully finished yet")

code/ID2TLib/StatsDatabase.py

@@ -298,7 +298,7 @@ class StatsDatabase:
 
         # Print results if option print_results is True
         if print_results:
-            if isinstance(result, list) and len(result) == 1:
+            if len(result) == 1 and isinstance(result, list):
                 result = result[0]
                 print("Query returned 1 record:\n")
                 for i in range(0, len(result)):

code_boost/src/cxx/pcap_processor.cpp

@@ -52,18 +52,10 @@ std::string pcap_processor::merge_pcaps(const std::string pcap_path) {
     std::string new_filepath = filePath;
     const std::string &newExt = "_" + tstmp + ".pcap";
     std::string::size_type h = new_filepath.rfind('.', new_filepath.length());
-
-    if ((filePath.length() + newExt.length()) < 250) {
-
-        if (h != std::string::npos) {
-            new_filepath.replace(h, newExt.length(), newExt);
-        } else {
-            new_filepath.append(newExt);
-        }
-    }
-
-    else {
-        new_filepath = (new_filepath.substr(0, new_filepath.find('_'))).append(newExt);
+    if (h != std::string::npos) {
+        new_filepath.replace(h, newExt.length(), newExt);
+    } else {
+        new_filepath.append(newExt);
     }
 
     FileSniffer sniffer_base(filePath);
@@ -166,6 +158,9 @@ void pcap_processor::collect_statistics() {
         
         // Save timestamp of last packet into statistics
         stats.setTimestampLastPacket(currentPktTimestamp);
+
+        // Create the communication interval statistics from all gathered communication intervals
+        stats.createCommIntervalStats();
     }
 }
 
@@ -252,15 +247,19 @@ void pcap_processor::process_packets(const Packet &pkt) {
         if (p == PDU::PDUType::TCP) {
             TCP tcpPkt = (const TCP &) *pdu_l4;
             
-          // Check TCP checksum
-          if (pdu_l3_type == PDU::PDUType::IP) {
-            stats.checkTCPChecksum(ipAddressSender, ipAddressReceiver, tcpPkt);
-          }
+            // Check TCP checksum
+            if (pdu_l3_type == PDU::PDUType::IP) {
+              stats.checkTCPChecksum(ipAddressSender, ipAddressReceiver, tcpPkt);
+            }
 
             stats.incrementProtocolCount(ipAddressSender, "TCP");                        
 
             // Conversation statistics
             stats.addConvStat(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp());
+            stats.addConvStatStateless(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp()); 
+
+            // Communication interval data collection for the later created communication statistics
+            stats.addCommInterval(ipAddressSender, tcpPkt.sport(), ipAddressReceiver, tcpPkt.dport(), pkt.timestamp());
 
             // Window Size distribution
             int win = tcpPkt.window();
@@ -279,8 +278,14 @@ void pcap_processor::process_packets(const Packet &pkt) {
           // UDP Packet
         } else if (p == PDU::PDUType::UDP) {
             const UDP udpPkt = (const UDP &) *pdu_l4;
-            stats.incrementProtocolCount(ipAddressSender, "UDP");            
-            stats.incrementPortCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport());                        
+            stats.incrementProtocolCount(ipAddressSender, "UDP");   
+
+            // Conversation statistics
+            stats.addConvStatStateless(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport(), pkt.timestamp());           
+            stats.incrementPortCount(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport());      
+
+            // Communication interval data collection for the later created communication statistics
+            stats.addCommInterval(ipAddressSender, udpPkt.sport(), ipAddressReceiver, udpPkt.dport(), pkt.timestamp());       
           
         } else if (p == PDU::PDUType::ICMP) {
             stats.incrementProtocolCount(ipAddressSender, "ICMP");

code_boost/src/cxx/statistics.cpp

@@ -247,6 +247,118 @@ void statistics::addConvStat(std::string ipAddressSender,int sport,std::string i
     }
 }
 
+/**
+ * Registers statistical data for a sent packet in a given stateless conversation (two IPs, two ports). 
+ * Increments the counter packets_A_B or packets_B_A.
+ * Adds the timestamp of the packet in pkts_A_B_timestamp or pkts_B_A_timestamp.
+ * @param ipAddressSender The sender IP address.
+ * @param sport The source port.
+ * @param ipAddressReceiver The receiver IP address.
+ * @param dport The destination port.
+ * @param timestamp The timestamp of the packet.
+ */
+void statistics::addConvStatStateless(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp){
+
+    conv f1 = {ipAddressReceiver, dport, ipAddressSender, sport};
+    conv f2 = {ipAddressSender, sport, ipAddressReceiver, dport};
+
+    // if already exist A(ipAddressReceiver, dport), B(ipAddressSender, sport) conversation
+    if (conv_statistics_stateless.count(f1)>0){
+        conv_statistics_stateless[f1].pkts_count++;
+        if(conv_statistics_stateless[f1].pkts_count<=3)
+            conv_statistics_stateless[f1].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics_stateless[f1].pkts_timestamp.back()));
+        conv_statistics_stateless[f1].pkts_timestamp.push_back(timestamp);
+    }
+    // Add new conversation A(ipAddressSender, sport), B(ipAddressReceiver, dport)
+    else{
+        conv_statistics_stateless[f2].pkts_count++;
+        if(conv_statistics_stateless[f2].pkts_timestamp.size()>0 && conv_statistics_stateless[f2].pkts_count<=3 )
+            conv_statistics_stateless[f2].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics_stateless[f2].pkts_timestamp.back()));
+        conv_statistics_stateless[f2].pkts_timestamp.push_back(timestamp);
+    }
+}
+
+/**
+ * Adds the passed information to the relevant communication intervals of the respective conversation.
+ * If the time between the last message of the latest interval and the timestamp of the current message exceeds
+ * the threshold, a new interval is created.
+ * Note: here and within the function, conversation refers to a stateless conversation.
+ * @param ipAddressSender The sender IP address.
+ * @param sport The source port.
+ * @param ipAddressReceiver The receiver IP address.
+ * @param dport The destination port.
+ * @param timestamp The timestamp of the packet.
+ */
+void statistics::addCommInterval(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp){
+    conv f1 = {ipAddressReceiver, dport, ipAddressSender, sport};
+    conv f2 = {ipAddressSender, sport, ipAddressReceiver, dport};
+    conv f;
+
+    // if there already exists a communication interval for the specified conversation ...
+    if (comm_intervals.count(f1) > 0 || comm_intervals.count(f2) > 0){
+
+        // find out which direction of conversation is contained in comm_intervals
+        if (comm_intervals.count(f1) > 0)
+            f = f1;
+        else
+            f = f2;
+
+        // if the time difference is exceeded, create a new interval with this message
+        if (timestamp - comm_intervals[f].back().end > (std::chrono::microseconds) ((unsigned long) COMM_INTERVAL_THRESHOLD)) {  // > or >= ?
+            commInterval new_interval = {timestamp, timestamp, 1};
+            comm_intervals[f].push_back(new_interval);
+        }  
+        // otherwise, set the time of the last interval message to the current timestamp and increase interval packet count by 1
+        else{
+            comm_intervals[f].back().end = timestamp;
+            comm_intervals[f].back().pkts_count++;
+        }
+    }
+    // if there does not exist a communication interval for the specified conversation ...
+    else{
+        // add initial interval for this conversation
+        commInterval initial_interval = {timestamp, timestamp, 1};
+
+        std::vector<commInterval> intervals;
+        intervals.push_back(initial_interval);
+        comm_intervals[f1] = intervals;
+    }
+}
+
+/**
+ * Aggregate the collected information about all communication intervals of every conversation.
+ * Do this by computing the average packet rate per interval and the average time between intervals.
+ * Note: here and within the function, conversation refers to a stateless conversation.
+ */
+void statistics::createCommIntervalStats(){    
+    // iterate over all <conv, conv_intervals> pairs
+    for (auto &cur_elem : comm_intervals) {
+        conv cur_conv = cur_elem.first;
+        std::vector<commInterval> intervals = cur_elem.second;
+
+        // if there is only one interval, the time between intervals cannot be computed and is therefore set to 0
+        if (intervals.size() == 1){
+            entry_commIntervalStat e = {(double) intervals[0].pkts_count, (double) 0};
+            comm_interval_statistics[cur_conv] = e;
+        }
+        // If there is more than one interval, compute the specified averages
+        else if (intervals.size() > 1){
+            long summed_pkts_count = intervals[0].pkts_count;
+            std::chrono::microseconds time_between_ints_sum = (std::chrono::microseconds) 0;
+
+            for (int i = 1; i < intervals.size(); i++) {
+                summed_pkts_count += intervals[i].pkts_count;
+                time_between_ints_sum += intervals[i].start - intervals[i - 1].end;
+            }
+
+            double avg_pkts_count = summed_pkts_count / ((double) intervals.size());
+            double avg_time_betw_ints = (time_between_ints_sum.count() / (double) (intervals.size() - 1)) / (double) 1e6;
+            entry_commIntervalStat e = {avg_pkts_count, avg_time_betw_ints};
+            comm_interval_statistics[cur_conv] = e;
+        }
+    }
+}
+
 /**
  * Increments the packet counter for the given IP address and MSS value.
  * @param ipAddress The IP address whose MSS packet counter should be incremented.
@@ -601,7 +713,9 @@ void statistics::writeToDatabase(std::string database_path) {
         db.writeStatisticsToS(tos_distribution);
         db.writeStatisticsWin(win_distribution);
         db.writeStatisticsConv(conv_statistics);
+        db.writeStatisticsConvStateless(conv_statistics_stateless);
         db.writeStatisticsInterval(interval_statistics);
+        db.writeCommIntervalStats(comm_interval_statistics);
     }
     else {
         // Tinslib failed to recognize the types of the packets in the input PCAP
@@ -609,42 +723,3 @@ void statistics::writeToDatabase(std::string database_path) {
         return;
     }
 }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

code_boost/src/cxx/statistics.h

@@ -15,6 +15,8 @@
 
 using namespace Tins;
 
+#define COMM_INTERVAL_THRESHOLD 10e6  // in microseconds; i.e. here 10s
+
 /*
  * Definition of structs used in unordered_map fields
  */
@@ -263,6 +265,40 @@ struct ipAddress_inOut_port {
     }
 };
 
+/*
+ * Struct used to represent a communication interval (for two hosts):
+ * - Timestamp of the first packet in the interval
+ * - Timestamp of the last packet in the interval
+ * - The count of packets within the interval
+ */
+struct commInterval{
+    std::chrono::microseconds start;
+    std::chrono::microseconds end;
+    long pkts_count;
+
+    bool operator==(const commInterval &other) const {
+        return start == other.start
+               && end == other.end
+               && pkts_count == other.pkts_count;
+    }    
+};
+
+/*
+ * Struct used to represent for the communication intervals of two hosts:
+ * - Average time between intervals
+ * - The average count of packets within an interval
+ */
+struct entry_commIntervalStat{
+    double avg_pkts_count;
+    double avg_time_between;
+
+    bool operator==(const entry_commIntervalStat &other) const {
+        return avg_pkts_count == other.avg_pkts_count
+               && avg_time_between == other.avg_time_between;
+    }    
+};
+
+
 /*
  * Definition of hash functions for structs used as key in unordered_map
  */
@@ -373,6 +409,12 @@ public:
 
     void addConvStat(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
 
+    void addConvStatStateless(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
+
+    void addCommInterval(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp);
+
+    void createCommIntervalStats();
+
     std::vector<float> calculateIPsCumEntropy();
 
     std::vector<float> calculateLastIntervalIPsEntropy(std::chrono::microseconds intervalStartTimestamp);
@@ -468,6 +510,7 @@ private:
     int intervalCumNovelMSSCount = 0;
     int intervalCumNovelPortCount = 0;
 
+
     /*
      * Data containers
      */
@@ -487,6 +530,11 @@ private:
     // average of inter-arrival times}
     std::unordered_map<conv, entry_convStat> conv_statistics;
 
+    // {IP Address A, Port A, IP Address B, Port B,   #packets, packets timestamps, inter-arrival times,
+    // average of inter-arrival times}
+    // Also stores conversation with only one exchanged message. In this case avgPktRate, minDelay, maxDelay and avgDelay are -1
+    std::unordered_map<conv, entry_convStat> conv_statistics_stateless;
+
     // {Last timestamp in the interval, #packets, #bytes, source IP entropy, destination IP entropy,
     // source IP cumulative entropy, destination IP cumulative entropy, #payload, #incorrect TCP checksum,
     // #correct TCP checksum, #novel IP, #novel TTL, #novel Window Size, #novel ToS,#novel MSS}
@@ -518,6 +566,13 @@ private:
 
     // {IP Address, MAC Address}
     std::unordered_map<std::string, std::string> ip_mac_mapping;
+
+    // {IP Address A, Port A, IP Address B, Port B, listof(commInterval)}
+    // Used to manage all communication intervals for a pair of communicating hosts
+    std::unordered_map<conv, std::vector<commInterval> > comm_intervals;
+
+    // {IP Address A, Port A, IP Address B, Port B, avg #packets, avg time between intervals}
+    std::unordered_map<conv, entry_commIntervalStat> comm_interval_statistics;
 };
 
 

code_boost/src/cxx/statistics_db.cpp

@@ -27,13 +27,13 @@ void statistics_db::writeStatisticsIP(std::unordered_map<std::string, entry_ipSt
         SQLite::Transaction transaction(*db);
         const char *createTable = "CREATE TABLE ip_statistics ( "
                 "ipAddress TEXT, "
-                "pktsReceived INTEGER, "
+                "pktsReceived INTEGtimestampER, "
                 "pktsSent INTEGER, "
                 "kbytesReceived REAL, "
                 "kbytesSent REAL, "
                 "maxPktRate REAL,"
                 "minPktRate REAL,"
-                "ipClass TEXT COLLATE NOCASE, "
+                "ipClass TEXT, "
                 "PRIMARY KEY(ipAddress));";
         db->exec(createTable);
         SQLite::Statement query(*db, "INSERT INTO ip_statistics VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
@@ -187,7 +187,7 @@ void statistics_db::writeStatisticsProtocols(std::unordered_map<ipAddress_protoc
         SQLite::Transaction transaction(*db);
         const char *createTable = "CREATE TABLE ip_protocols ("
                 "ipAddress TEXT,"
-                "protocolName TEXT COLLATE NOCASE,"
+                "protocolName TEXT,"
                 "protocolCount INTEGER,"
                 "PRIMARY KEY(ipAddress,protocolName));";
         db->exec(createTable);
@@ -217,7 +217,7 @@ void statistics_db::writeStatisticsPorts(std::unordered_map<ipAddress_inOut_port
         SQLite::Transaction transaction(*db);
         const char *createTable = "CREATE TABLE ip_ports ("
                 "ipAddress TEXT,"
-                "portDirection TEXT COLLATE NOCASE,"
+                "portDirection TEXT,"
                 "portNumber INTEGER,"
                 "portCount INTEGER,"
                 "PRIMARY KEY(ipAddress,portDirection,portNumber));";
@@ -249,7 +249,7 @@ void statistics_db::writeStatisticsIpMac(std::unordered_map<std::string, std::st
         SQLite::Transaction transaction(*db);
         const char *createTable = "CREATE TABLE ip_mac ("
                 "ipAddress TEXT,"
-                "macAddress TEXT COLLATE NOCASE,"
+                "macAddress TEXT,"
                 "PRIMARY KEY(ipAddress));";
         db->exec(createTable);
         SQLite::Statement query(*db, "INSERT INTO ip_mac VALUES (?, ?)");
@@ -380,6 +380,91 @@ void statistics_db::writeStatisticsConv(std::unordered_map<conv, entry_convStat>
     }
 }
 
+
+/**
+ * Writes the stateless conversation statistics into the database.
+ * @param convStatistics The stateless conversation from class statistics.
+ */
+void statistics_db::writeStatisticsConvStateless(std::unordered_map<conv, entry_convStat> convStatistics){          
+    try {
+        db->exec("DROP TABLE IF EXISTS conv_statistics_stateless");
+        SQLite::Transaction transaction(*db);
+        const char *createTable = "CREATE TABLE conv_statistics_stateless ("
+                "ipAddressA TEXT,"
+                "portA INTEGER,"
+                "ipAddressB TEXT,"              
+                "portB INTEGER,"
+                "pktsCount INTEGER,"
+                "avgPktRate REAL,"
+                "avgDelay INTEGER,"
+                "minDelay INTEGER,"
+                "maxDelay INTEGER,"
+                "PRIMARY KEY(ipAddressA,portA,ipAddressB,portB));";
+        db->exec(createTable);
+        SQLite::Statement query(*db, "INSERT INTO conv_statistics_stateless VALUES (?, ?, ?, ?, ?,  ?, ?, ?, ?)");
+
+        // Calculate average of inter-arrival times and average packet rate
+        for (auto it = convStatistics.begin(); it != convStatistics.end(); ++it) {
+            conv f = it->first;
+            entry_convStat e = it->second;
+            if (e.pkts_count > 0){
+                query.bind(1, f.ipAddressA);
+                query.bind(2, f.portA);
+                query.bind(3, f.ipAddressB);
+                query.bind(4, f.portB);
+
+                if (e.pkts_count == 1){
+                    e.avg_pkt_rate = (float) -1;
+                    e.avg_interarrival_time = (std::chrono::microseconds) -1;
+
+                    query.bind(5, (int) e.pkts_count);
+                    query.bind(6, (float) e.avg_pkt_rate);
+                    query.bind(7, (int) e.avg_interarrival_time.count());
+                    query.bind(8, -1);
+                    query.bind(9, -1);
+                    query.exec();
+                    query.reset();
+                }
+                else {
+                    int sumDelay = 0;
+                    int minDelay = -1;
+                    int maxDelay = -1;
+                    for (int i = 0; (unsigned) i < e.interarrival_time.size(); i++) {
+                        sumDelay += e.interarrival_time[i].count();
+                        if (maxDelay < e.interarrival_time[i].count())
+                            maxDelay = e.interarrival_time[i].count();
+                        if (minDelay > e.interarrival_time[i].count() || minDelay == -1)
+                            minDelay = e.interarrival_time[i].count();
+                    }
+                    if (e.interarrival_time.size() > 0)
+                        e.avg_interarrival_time = (std::chrono::microseconds) sumDelay / e.interarrival_time.size(); // average
+                    else e.avg_interarrival_time = (std::chrono::microseconds) 0;
+
+                    std::chrono::microseconds start_timesttamp = e.pkts_timestamp[0];
+                    std::chrono::microseconds end_timesttamp = e.pkts_timestamp.back();
+                    std::chrono::microseconds conn_duration = end_timesttamp - start_timesttamp;
+                    e.avg_pkt_rate = (float) e.pkts_count * 1000000 / conn_duration.count(); // pkt per sec
+
+                    
+                    query.bind(5, (int) e.pkts_count);
+                    query.bind(6, (float) e.avg_pkt_rate);
+                    query.bind(7, (int) e.avg_interarrival_time.count());
+                    query.bind(8, minDelay);
+                    query.bind(9, maxDelay);
+                    query.exec();
+                    query.reset();
+                }
+            }
+            
+        }
+        transaction.commit();
+    }
+    catch (std::exception &e) {
+        std::cout << "Exception in statistics_db: " << e.what() << std::endl;
+    }
+}
+
+
 /**
  * Writes the interval statistics into the database.
  * @param intervalStatistics The interval entries from class statistics.
@@ -438,3 +523,45 @@ void statistics_db::writeStatisticsInterval(std::unordered_map<std::string, entr
     }
 }
 
+/**
+ * Writes the communication interval statistics for every conversation into the database.
+ * @param commIntervalStatistics The communication interval statistics from class statistics.
+ */
+void statistics_db::writeCommIntervalStats(std::unordered_map<conv, entry_commIntervalStat> commIntervalStatistics){
+    try {
+        db->exec("DROP TABLE IF EXISTS comm_interval_statistics");
+        SQLite::Transaction transaction(*db);
+        const char *createTable = "CREATE TABLE comm_interval_statistics ("
+                "ipAddressA TEXT,"
+                "portA INTEGER,"
+                "ipAddressB TEXT,"              
+                "portB INTEGER,"
+                "avgPktCount REAL,"
+                "avgTimeBetweenIntervals REAL,"
+                "PRIMARY KEY(ipAddressA,portA,ipAddressB,portB));";
+        db->exec(createTable);
+        SQLite::Statement query(*db, "INSERT INTO comm_interval_statistics VALUES (?, ?, ?, ?, ?, ?)");
+
+        // iterate over every conversation and interval aggregation pair and store the respective values in the database
+        for (auto it = commIntervalStatistics.begin(); it != commIntervalStatistics.end(); ++it) {
+            conv f = it->first;
+            entry_commIntervalStat e = it->second;
+            if (e.avg_pkts_count > 0){
+                query.bind(1, f.ipAddressA);
+                query.bind(2, f.portA);
+                query.bind(3, f.ipAddressB);
+                query.bind(4, f.portB);
+                query.bind(5, e.avg_pkts_count);
+                query.bind(6, e.avg_time_between);
+                
+                query.exec();
+                query.reset();
+            }
+            
+        }
+        transaction.commit();
+    }
+    catch (std::exception &e) {
+        std::cout << "Exception in statistics_db: " << e.what() << std::endl;
+    }
+}

code_boost/src/cxx/statistics_db.h

@@ -43,8 +43,12 @@ public:
 
     void writeStatisticsConv(std::unordered_map<conv, entry_convStat> convStatistics);
 
+    void writeStatisticsConvStateless(std::unordered_map<conv, entry_convStat> convStatistics);
+
     void writeStatisticsInterval(std::unordered_map<std::string, entry_intervalStat> intervalStatistics);
 
+    void writeCommIntervalStats(std::unordered_map<conv, entry_commIntervalStat> commIntervalStatistics);
+
 private:
     // Pointer to the SQLite database
     std::unique_ptr<SQLite::Database> db;