Merge branch 'develop' of https://git.tk.informatik.tu-darmstadt.de/leon.boeck/ID2T-toolkit-BotnetTraffic into develop

code/Attack/MembersMgmtCommAttack.py

@@ -58,13 +58,9 @@ class Message():
         return str_
 
 
-from ID2TLib import FileUtils, PaddingGenerator
-from ID2TLib.PacketGenerator import PacketGenerator
-from ID2TLib.IPGenerator import IPGenerator
+from ID2TLib import FileUtils, Generator
 from ID2TLib.PcapAddressOperations import PcapAddressOperations
 from ID2TLib.CommunicationProcessor import CommunicationProcessor
-from ID2TLib.MacAddressGenerator import MacAddressGenerator
-from ID2TLib.PortGenerator import gen_random_server_port
 from ID2TLib.Botnet.MessageMapping import MessageMapping
 
 
@@ -159,7 +155,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
 
         # Setup (initial) parameters for packet creation loop
         BUFFER_SIZE = 1000
-        pkt_gen = PacketGenerator()
+        pkt_gen = Generator.PacketGenerator()
         padding = self.get_param_value(Param.PACKET_PADDING)
         packets = deque(maxlen=BUFFER_SIZE)
         total_pkts = 0
@@ -195,7 +191,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             # create suitable IP/UDP packet and add to packets list
             packet = pkt_gen.generate_mmcom_packet(ip_src=ip_src, ip_dst=ip_dst, ttl=ttl, mac_src=mac_src, mac_dst=mac_dst, 
                 port_src=port_src, port_dst=port_dst, message_type=msg.type, neighborlist_entries=nl_size)
-            PaddingGenerator.add_padding(packet, padding,True, True)
+            Generator.add_padding(packet, padding,True, True)
 
             packet.time = msg.time
             packets.append(packet)
@@ -207,7 +203,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
                 self.attack_start_utime = packets[0].time
             elif total_pkts % BUFFER_SIZE == 0: # every 1000 packets write them to the PCAP file (append)
                 packets = list(packets)
-                PaddingGenerator.equal_length(packets, padding = padding)
+                Generator.equal_length(packets, padding = padding)
                 last_packet = packets[-1]
                 path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
                 packets = deque(maxlen=BUFFER_SIZE)
@@ -215,7 +211,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         # if there are unwritten packets remaining, write them to the PCAP file
         if len(packets) > 0:
             packets = list(packets)
-            PaddingGenerator.equal_length(packets, padding = padding)
+            Generator.equal_length(packets, padding = padding)
             path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
             last_packet = packets[-1]
 
@@ -245,7 +241,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             ids = ids_to_add.copy()
             # macgen only needed, when IPs are new local IPs (therefore creating the object here suffices for the current callers
             # to not end up with the same MAC paired with different IPs)
-            macgen = MacAddressGenerator()
+            macgen = Generator.MacAddressGenerator()
 
             # assign existing IPs and the corresponding MAC addresses in the PCAP to the IDs
             for ip in existing_ips:
@@ -360,7 +356,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         reuse_count_local = int(reuse_percent_total * reuse_percent_local * len(mapped_ids))
 
         # create locality, IP and MAC configurations for the IDs/Bots
-        ipgen = IPGenerator()
+        ipgen = Generator.IPGenerator()
         pcapops = PcapAddressOperations(self.statistics)
         router_mac = pcapops.get_probable_router_mac()
         bot_configs = {}
@@ -447,7 +443,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
                     
         # create port configurations for the bots
         for bot in bot_configs:
-            bot_configs[bot]["Port"] = gen_random_server_port()    
+            bot_configs[bot]["Port"] = Generator.gen_random_server_port()
 
         # print(local_init_ids)
         # print(bot_configs)

code/ID2TLib/FileUtils.py

@@ -48,8 +48,8 @@ def parse_csv_to_xml(filepath: str):
 			for element in line:
 				element = element.replace(" ", "")
 				key, value = element.split(":")
-				packet.attrib[key] = value
-			packet.attrib["LineNumber"] = lineno
+				packet.attrib[key] = str(value)
+			packet.attrib["LineNumber"] = str(lineno)
 
 	# writing the ElementTree into the .xml file
 	tree = ElementTree.ElementTree(root)

code/ID2TLib/Generator.py

@@ -0,0 +1,393 @@
+from scapy.packet import Raw
+
+import numpy.random as random2
+import random
+import string
+
+from numpy.random import bytes
+from random import getrandbits
+from scapy.layers.inet import IP, Ether, UDP, TCP
+from scapy.packet import Raw
+from Attack.MembersMgmtCommAttack import MessageType
+from . import IPv4 as ip
+
+
+
+
+'''PaddingGenerator
+'''
+
+def add_padding(packet, bytes_padding = 0, user_padding=True, rnd = False):
+    '''
+    Adds padding to a packet with the given amount of bytes, but a maximum of 100 bytes.
+    :param packet: the packet that will be extended with the additional payload
+    :param bytes_padding: the amount of bytes that will be appended to the packet
+    :param user_padding: true, if the function add_padding is called from another class and the user
+    sets the padding manually
+    :param rnd: adds a random padding betwing 0 and bytes_adding, if true
+    :return: the initial packet, extended with the wanted amount of bytes of padding
+    '''
+
+    if(user_padding and bytes_padding > 100):
+        bytes_padding = 100
+
+    if (rnd is True):
+        r = int(round(bytes_padding / 4))                  #sets bytes_padding to any number between 0 and bytes_padding
+        bytes_padding = random2.random_integers(0, r) * 4   #, that's dividable by 4
+    payload = generate_payload(bytes_padding)
+    packet[Raw].load += Raw(load=payload).load
+    return packet
+
+def equal_length(list_of_packets, length = 0, padding = 0):
+    '''
+    Equals the length of a given set of packets on the given length. If the given length is smaller than the largest
+    packet, all the other packets are extended to the largest packet's length.
+    :param list_of_packets: The given set of packet.
+    :param length: The length each packet should have.
+    :return: The set of extended packets.
+    '''
+
+    largest_packet = length
+    for packet in list_of_packets:
+        packet_length = len(packet)
+        if(packet_length > largest_packet):
+            largest_packet = packet_length
+
+    for packet in list_of_packets:
+        bytes_padding = largest_packet - len(packet)
+        if(bytes_padding > 0):
+            add_padding(packet, bytes_padding, False, False)
+            add_padding(packet, padding, False, True)
+
+    return list_of_packets
+
+'''PayloadGenerator
+'''
+
+def generate_payload(size:int=0):
+
+	"""
+	Generates a payload of random bytes of the given amount
+
+	:param size: number of generated bytes
+    :return: the generated payload
+	"""
+
+	payload = bytes(size)
+
+	return payload
+
+
+'''PortGenerator
+'''
+
+def gen_random_server_port(offset: int=2199):
+    """
+    Generates a valid random first and last character for a bots hostname
+    and computes a port from these two characters.
+    The default offset is chosen from a Sality implementation in 2011
+    """
+    firstLetter = random.choice(string.ascii_letters);
+    lastLetter = random.choice(string.ascii_letters + string.digits);
+    return (offset + ord(firstLetter) * ord(lastLetter));
+
+
+'''MacAddressGenerator
+'''
+
+
+class MacAddressGenerator:
+    def __init__(self, include_broadcast_macs=False, include_virtual_macs=False):
+        self.broadcast = include_broadcast_macs
+        self.virtual = include_virtual_macs
+
+        self.generated = set()
+
+    def random_mac(self) -> str:
+        while True:
+            mac = self._random_mac()
+            if mac not in self.generated:
+                self.generated.add(mac)
+                return mac
+
+    def clear(self):
+        self.generated.clear()
+
+    def generates_broadcast_macs(self) -> bool:
+        return self.broadcast
+
+    def generates_virtual_macs(self) -> bool:
+        return self.virtual
+
+    def set_broadcast_generation(self, broadcast: bool):
+        self.broadcast = broadcast
+
+    def set_virtual_generation(self, virtual: bool):
+        self.virtual = virtual
+
+    def _random_mac(self) -> str:
+        mac_bytes = bytearray(getrandbits(8) for i in range(6))
+        if not self.broadcast:
+            mac_bytes[0] &= ~1  # clear the first bytes' first bit
+        if not self.virtual:
+            mac_bytes[0] &= ~2  # clear the first bytes' second bit
+
+        return ":".join("%02X" % b for b in mac_bytes)
+
+
+
+'''PacketGenerator
+'''
+
+
+class PacketGenerator():
+    """
+    Creates packets, based on the set protocol
+    """
+
+    def __init__(self, protocol="udp"):
+        """
+        Creates a new Packet_Generator Object
+
+        :param protocol: the protocol of the packets to be created, udp or tcp
+        """
+        super(PacketGenerator, self).__init__()
+        self.protocol = protocol
+
+    def generate_packet(self, ip_src: str = "192.168.64.32", ip_dst: str = "192.168.64.48",
+                        mac_src: str = "56:6D:D9:BC:70:1C",
+                        mac_dst: str = "F4:2B:95:B3:0E:1A", port_src: int = 1337, port_dst: int = 6442, ttl: int = 64,
+                        tcpflags: str = "S", payload: str = ""):
+        """
+        Creates a Packet with the specified Values for the current protocol
+
+        :param ip_src: the source IP address of the IP header
+        :param ip_dst the destination IP address of the IP header
+        :param mac_src: the source MAC address of the MAC header
+        :param mac_dst: the destination MAC address of the MAC header
+        :param port_src: the source port of the header
+        :param port_dst: the destination port of the header
+        :param ttl: the ttl Value of the packet
+        :param tcpflags: the TCP flags of the TCP header
+        :param payload: the payload of the packet
+        :return: the corresponding packet
+        """
+
+        if (self.protocol == "udp"):
+            packet = generate_udp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+                                         port_src=port_src, port_dst=port_dst, payload=payload)
+        elif (self.protocol == "tcp"):
+            packet = generate_tcp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+                                         port_src=port_src, port_dst=port_dst, tcpflags=tcpflags, payload=payload)
+        return packet
+
+    def generate_mmcom_packet(self, ip_src: str = "192.168.64.32", ip_dst: str = "192.168.64.48",
+                              mac_src: str = "56:6D:D9:BC:70:1C",
+                              mac_dst: str = "F4:2B:95:B3:0E:1A", port_src: int = 1337, port_dst: int = 6442,
+                              tcpflags: str = "S", ttl: int = 64,
+                              message_type: MessageType = MessageType.SALITY_HELLO, neighborlist_entries: int = 1):
+        """
+        Creates a Packet for Members-Management-Communication with the specified Values and the current protocol
+
+        :param ip_src: the source IP address of the IP header
+        :param ip_dst the destination IP address of the IP header
+        :param mac_src: the source MAC address of the MAC header
+        :param mac_dst: the destination MAC address of the MAC header
+        :param port_src: the source port of the header
+        :param port_dst: the destination port of the header
+        :param tcpflags: the TCP flags of the TCP header, if tcp is selected as protocol
+        :param ttl: the ttl Value of the packet
+        :param message_type: affects the size of the payload
+        :param neighborlist_entries: number of entries of a Neighbourlist-reply, affects the size of the payload
+        :return: the corresponding packet
+        """
+
+        # Determine length of the payload that has to be generated
+        if (message_type == MessageType.SALITY_HELLO):
+            payload_len = 0
+        elif (message_type == MessageType.SALITY_HELLO_REPLY):
+            payload_len = 22
+        elif (message_type == MessageType.SALITY_NL_REQUEST):
+            payload_len = 28
+        elif (message_type == MessageType.SALITY_NL_REPLY):
+            payload_len = 24 + 6 * neighborlist_entries
+        else:
+            payload_len = 0
+
+        payload = generate_payload(payload_len)
+
+        if (self.protocol == "udp"):
+            packet = generate_udp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+                                         port_src=port_src, port_dst=port_dst, payload=payload)
+        elif (self.protocol == "tcp"):
+            packet = generate_tcp_packet(ip_src=ip_src, ip_dst=ip_dst, mac_src=mac_src, mac_dst=mac_dst, ttl=ttl,
+                                         port_src=port_src, port_dst=port_dst, tcpflags=tcpflags, payload=payload)
+        else:
+            print("Error: unsupported protocol for generating Packets")
+
+        return packet
+
+
+def generate_tcp_packet(ip_src: str = "192.168.64.32", ip_dst: str = "192.168.64.48",
+                        mac_src: str = "56:6D:D9:BC:70:1C", ttl: int = 64,
+                        mac_dst: str = "F4:2B:95:B3:0E:1A", port_src: int = 1337, port_dst: int = 6442,
+                        tcpflags: str = "S", payload: str = ""):
+    """
+    Builds a TCP packet with the values specified by the caller.
+
+    :param ip_src: the source IP address of the IP header
+    :param ip_dst the destination IP address of the IP header
+    :param mac_src: the source MAC address of the MAC header
+    :param ttl: the ttl value of the packet
+    :param mac_dst: the destination MAC address of the MAC header
+    :param port_src: the source port of the TCP header
+    :param port_dst: the destination port of the TCP header
+    :param tcpflags: the TCP flags of the TCP header
+    :param payload: the payload of the packet
+    :return: the corresponding TCP packet
+    """
+
+    ether = Ether(src=mac_src, dst=mac_dst)
+    ip = IP(src=ip_src, dst=ip_dst, ttl=ttl)
+    tcp = TCP(sport=port_src, dport=port_dst, flags=tcpflags)
+    packet = ether / ip / tcp / Raw(load=payload)
+    return packet
+
+
+def generate_udp_packet(ip_src: str = "192.168.64.32", ip_dst: str = "192.168.64.48",
+                        mac_src: str = "56:6D:D9:BC:70:1C", ttl: int = 64,
+                        mac_dst: str = "F4:2B:95:B3:0E:1A", port_src: int = 1337, port_dst: int = 6442,
+                        payload: str = ""):
+    """
+    Builds an UDP packet with the values specified by the caller.
+
+    :param ip_src: the source IP address of the IP header
+    :param ip_dst the destination IP address of the IP header
+    :param mac_src: the source MAC address of the MAC header
+    :param ttl: the ttl value of the packet
+    :param mac_dst: the destination MAC address of the MAC header
+    :param port_src: the source port of the UDP header
+    :param port_dst: the destination port of the UDP header
+    :param payload: the payload of the packet
+    :return: the corresponding UDP packet
+    """
+
+    ether = Ether(src=mac_src, dst=mac_dst)
+    ip = IP(src=ip_src, dst=ip_dst, ttl=ttl)
+    udp = UDP(sport=port_src, dport=port_dst)
+    packet = ether / ip / udp / Raw(load=payload)
+    return packet
+
+'''IPGenerator
+'''
+
+
+class IPChooser:
+    def random_ip(self):
+        return ip.IPAddress.from_int(random.randrange(0, 1 << 32))
+
+    def size(self):
+        return 1 << 32
+
+    def __len__(self):
+        return self.size()
+
+
+class IPChooserByRange(IPChooser):
+    def __init__(self, ip_range):
+        self.range = ip_range
+
+    def random_ip(self):
+        start = int(self.range.first_address())
+        end = start + self.range.block_size()
+        return ip.IPAddress.from_int(random.randrange(start, end))
+
+    def size(self):
+        return self.range.block_size()
+
+
+class IPChooserByList(IPChooser):
+    def __init__(self, ips):
+        self.ips = list(ips)
+        if not self.ips:
+            raise ValueError("list of ips must not be empty")
+
+    def random_ip(self):
+        return random.choice(self.ips)
+
+    def size(self):
+        return len(self.ips)
+
+
+class IPGenerator:
+    def __init__(self, ip_chooser=IPChooser(),  # include all ip-addresses by default (before the blacklist)
+                 include_private_ips=False, include_localhost=False,
+                 include_multicast=False, include_reserved=False,
+                 include_link_local=False, blacklist=None):
+        self.blacklist = []
+        self.generated_ips = set()
+
+        if not include_private_ips:
+            for segment in ip.ReservedIPBlocks.PRIVATE_IP_SEGMENTS:
+                self.add_to_blacklist(segment)
+        if not include_localhost:
+            self.add_to_blacklist(ip.ReservedIPBlocks.LOCALHOST_SEGMENT)
+        if not include_multicast:
+            self.add_to_blacklist(ip.ReservedIPBlocks.MULTICAST_SEGMENT)
+        if not include_reserved:
+            self.add_to_blacklist(ip.ReservedIPBlocks.RESERVED_SEGMENT)
+        if not include_link_local:
+            self.add_to_blacklist(ip.ReservedIPBlocks.ZERO_CONF_SEGMENT)
+        if blacklist:
+            for segment in blacklist:
+                self.add_to_blacklist(segment)
+        self.chooser = ip_chooser
+
+    @staticmethod
+    def from_range(range, *args, **kwargs):
+        return IPGenerator(IPChooserByRange(range), *args, **kwargs)
+
+    def add_to_blacklist(self, ip_segment):
+        if isinstance(ip_segment, ip.IPAddressBlock):
+            self.blacklist.append(ip_segment)
+        else:
+            self.blacklist.append(ip.IPAddressBlock.parse(ip_segment))
+
+    def random_ip(self):
+        if len(self.generated_ips) == self.chooser.size():
+            raise ValueError("Exhausted the space of possible ip-addresses, no new unique ip-address can be generated")
+
+        while True:
+            random_ip = self.chooser.random_ip()
+
+            if not self._is_in_blacklist(random_ip) and random_ip not in self.generated_ips:
+                self.generated_ips.add(random_ip)
+                return str(random_ip)
+
+    def clear(self, clear_blacklist=True, clear_generated_ips=True):
+        if clear_blacklist: self.blacklist.clear()
+        if clear_generated_ips: self.generated_ips.clear()
+
+    def _is_in_blacklist(self, ip: ip.IPAddress):
+        return any(ip in block for block in self.blacklist)
+
+
+class MappingIPGenerator(IPGenerator):
+    def __init__(self, *args, **kwargs):
+        super().__init__(self, *args, **kwargs)
+
+        self.mapping = {}
+
+    def clear(self, clear_generated_ips=True, *args, **kwargs):
+        super().clear(self, clear_generated_ips=clear_generated_ips, *args, **kwargs)
+        if clear_generated_ips:
+            self.mapping = {}
+
+    def get_mapped_ip(self, key):
+        if key not in self.mapping:
+            self.mapping[key] = self.random_ip()
+
+        return self.mapping[key]
+
+    def __getitem__(self, item):
+        return self.get_mapped_ip(item)

code/ID2TLib/LabelManager.py

@@ -2,7 +2,38 @@ import os.path
 from datetime import datetime
 from xml.dom.minidom import *
 
-import ID2TLib.Label as Label
+from functools import total_ordering
+
+@total_ordering
+class Label:
+    def __init__(self, attack_name, timestamp_start, timestamp_end, attack_note=""):
+        """
+        Creates a new attack label
+
+        :param attack_name: The name of the associated attack
+        :param timestamp_start: The timestamp as unix time of the first attack packet
+        :param timestamp_end: The timestamp as unix time of the last attack packet
+        :param attack_note: A note associated to the attack (optional)
+        """
+        self.attack_name = attack_name
+        self.timestamp_start = timestamp_start
+        self.timestamp_end = timestamp_end
+        self.attack_note = attack_note
+
+    def __eq__(self, other):
+        return self.timestamp == other.timestamp
+
+    def __lt__(self, other):
+        return self.timestamp_start < other.timestamp_start
+
+    def __gt__(self, other):
+        return self.timestamp_start > other.timestamp_start
+
+    def __str__(self):
+        return ''.join(
+            ['(', self.attack_name, ',', self.attack_note, ',', str(self.timestamp_start), ',', str(self.timestamp_end),
+             ')'])
+
 
 
 class LabelManager:

code/ID2TLib/IPGenerator.py → code/ID2TLib/OldLibs/IPGenerator.py

@@ -1,5 +1,5 @@
 import random
-from . import IPv4 as ip
+from code.ID2TLib import IPv4 as ip
 
 class IPChooser:
 	def random_ip(self):

code/ID2TLib/Ports.py

@@ -24,11 +24,11 @@ class PortSelectionStrategy:
 		
 		# that function will always return a one higher counter than before,
 		# restarting from the start once it reached the highest value
-		def __call__(port_range):
+		def __call__(self, port_range, *args):
 			if self.counter == -1:
 				self.counter = port_range.start
 			
-			port = counter
+			port = self.counter
 			
 			self.counter += 1
 			if self.counter == port_range.stop:
@@ -36,9 +36,90 @@ class PortSelectionStrategy:
 			
 			return port
 	class random:
-		def __call__(port_range):
+		def __call__(port_range, *args):
 			return random.randrange(port_range.start, port_range.stop)
 
+	class linux_kernel:
+		"""
+		A port-selectioin-strategy oriented on the linux-kernel
+		The implementation follows https://github.com/torvalds/linux/blob/master/net/ipv4/inet_connection_sock.c#L173
+		as much as possible when converting from one language to another (The newest file was used
+		by the time of writing, make sure you select the correct one when following the link!)
+		"""
+
+		def __call__(self, port_range: range, port_selector, *args):
+			"""
+			This method is an attempt to map a c-function to python. To solve the goto-problem
+			while-true's have been added. Both of the while-true's are placed where the original
+			had a label to jump to. break's and continue's are set to preserve the original
+			control flow. Another method could have been used to rewrite the c-code, however this
+			was chosen to preserve the similarity between this and the original
+
+			:param port_range: the port range to choose from
+			:param port_selector: the port selector that tells which ports are in use
+			:param args: Not used for now
+			:return: A port number
+			"""
+			port = 0
+			low, high = port_range.start, port_range.stop
+
+			# this var tells us if we should use the upper or lower port-range-half, or the whole range if
+			# this var is None. The original was an enum of the values 0, 1 and 2. But I think an Optional[bool]
+			# is more clear
+			# None: use whole range, True: use lower half, False: use upper half
+			attempt_half = True
+
+			high += 1  # line 186 in the original file
+			while True:
+				if high - low < 4:
+					attempt_half = None
+				if attempt_half is not None:
+					# appearently a fast method to find a number close to the real half
+					# unless the difference between high and low is 4 (see above, note the 2-shift below)
+					# this does not work
+					half = low + (((high - low) >> 2) << 1)
+
+					if attempt_half:
+						high = half
+					else:
+						low = half
+
+				remaining = high - low
+				if remaining > 1:
+					remaining &= ~1 # flip the 1-bit
+
+				offset = random.randrange(0, remaining)
+				offset |= 1;
+
+				attempt_half_before = attempt_half # slight hack to keep track of change
+				while True:
+					port = low + offset
+
+					for i in range(0, remaining, 2):
+						if port >= high:
+							port -= remaining
+
+						if port_selector.is_port_in_use(port):
+							port += 2
+							continue
+
+						return port
+
+					offset -= 1
+					if not (offset & 1):
+						continue
+
+					if attempt_half:
+						attempt_half = False
+						break
+
+				if attempt_half_before: # we still got ports to search, attemp_half was just set to False
+					continue
+				if not attempt_half: # the port-range is exhausted
+					break
+
+			raise ValueError("Could not find suitable port")
+
 class PortSelector:
 	"""
 	This class simulates a port-selection-process. Instances keep a list of port-numbers they generated so
@@ -55,7 +136,7 @@ class PortSelector:
 		
 		if len(port_range) == 0:
 			raise ValueError("cannot choose from an empty range")
-		if x.start not in range(1, 65536) or x.stop not in range(1, 65536):
+		if port_range.start not in range(1, 65536) or port_range.stop not in range(1, 65536 + 1):
 			raise ValueError("port_range is no subset of the valid port-range")
 		
 		self.port_range = port_range
@@ -70,7 +151,7 @@ class PortSelector:
 			raise RuntimeError("All %i port numbers were already generated, no more can be generated" % len(self.port_range))
 		
 		while True:
-			port = self._select_port(self.port_range)
+			port = self._select_port(self.port_range, self)
 			
 			if port not in self.generated:
 				self.generated.append(port)

code/ID2TLib/Statistics.py

@@ -10,6 +10,7 @@ matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 from ID2TLib.PcapFile import PcapFile
 from ID2TLib.StatsDatabase import StatsDatabase
+from ID2TLib.IPv4 import IPAddress
 
 
 class Statistics:
@@ -544,6 +545,108 @@ class Statistics:
         else:
             return None
 
+    def get_in_degree(self):
+        """
+        determines the in-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has received packets from
+        :return: a list, each entry consists of one local IPAddress and its associated in-degree
+        """
+
+        in_degree_raw = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'in\' AND portNumber = portA GROUP BY ipAddress " +
+                "UNION " +
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'in\' AND portNumber = portB GROUP BY ipAddress")
+        
+        #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
+        in_degree = self.filter_multiples(in_degree_raw)
+
+        return in_degree
+
+    def get_out_degree(self):
+        """
+        determines the out-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
+        :return: a list, each entry consists of one local IPAddress and its associated out-degree
+        """
+        """
+
+        test = self.stats_db._process_user_defined_query("SELECT DISTINCT * FROM conv_statistics")
+        #test2 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddressB, portB FROM conv_statistics")
+        print("############# conv_statistics IP's + Ports")
+        for p in test:
+            print(p)
+        #for p in test2:
+        #    print(p)
+
+        print("############## ip_ports ##################")
+        test3 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddress, portNumber, portDirection FROM ip_ports")
+        for p in test3:
+            print(p)
+
+        print("")
+        print("############## AFTER JOIN - A #############")
+        test4 = self.stats_db._process_user_defined_query(
+                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA") # Hier werden die anfang locals rausgefiltert!
+        for p in test4:
+            print(p)
+
+        print("")
+        print("############## AFTER JOIN - B #############")
+        test6 = self.stats_db._process_user_defined_query(
+                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB") # Hier werden die anfang locals rausgefiltert!
+        for p in test6:
+            print(p)
+
+        print("")
+        print("############## BUILD UP PART FOR PART#############")
+        test5 = self.stats_db._process_user_defined_query(
+                "SELECT ipAddress, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' GROUP BY ipAddress")
+        for p in test5:
+            print(p)
+        """
+        out_degree_raw = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA GROUP BY ipAddress " +
+                "UNION " +
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB GROUP BY ipAddress")
+
+        #filter out non-local IPs
+        #out_degree_raw_2 = []
+        #for entry in out_degree_raw:
+        #    if IPAddress.parse(entry[0]).is_reserved():
+        #        out_degree_raw_2.append(entry)
+
+        #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
+        out_degree = self.filter_multiples(out_degree_raw)
+
+        return out_degree
+
+    def filter_multiples(self, entries):
+        """
+        helper function, for get_out_degree and get_in_degree
+        filters the given list for duplicate IpAddresses and, if duplciates are present, accumulates their values
+
+        :param entries: list, each entry consists of an ipAddress and a numeric value
+        :return: a filtered list, without duplicate ipAddresses
+        """
+
+        filtered_entries = []
+        done = []
+        for p1 in entries:       
+            added = False
+            if p1 in done:
+                continue
+            for p2 in entries:
+                if p1[0] == p2[0] and p1 != p2:
+                    filtered_entries.append((p1[0], p1[1] + p2[1]))
+                    done.append(p1)
+                    done.append(p2)
+                    #entries.remove(p2)
+                    added = True
+                    break
+
+            if not added:
+                filtered_entries.append(p1)
+
+        return filtered_entries
+
 
     def get_statistics_database(self):
         """
@@ -925,6 +1028,113 @@ class Statistics:
                 plt.savefig(out, dpi=500)
                 return out
 
+        def plot_packets_per_connection(file_ending: str):
+            plt.gcf().clear()
+            result = self.stats_db._process_user_defined_query(
+                "SELECT ipAddressA, portA, ipAddressB, portB, pktsCount FROM conv_statistics")
+            if (result):
+                graphy, graphx = [], []
+                # plot data in descending order
+                result = sorted(result, key=lambda row: row[4])
+
+                # compute plot data
+                for i, row in enumerate(result):
+                    addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
+                    # adjust the justification of strings to improve appearance
+                    len_max = max(len(addr1), len(addr2))
+                    addr1 = addr1.ljust(len_max)
+                    addr2 = addr2.ljust(len_max)
+                    # add plot data
+                    graphy.append("%s\n%s" % (addr1, addr2))
+                    graphx.append(row[4])
+
+                # compute plot height in inches
+                dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
+                plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
+                title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8 
+
+                # have x axis and its label appear at the top (instead of bottom)
+                fig, ax = plt.subplots()
+                ax.xaxis.tick_top()
+                ax.xaxis.set_label_position("top")
+
+                # set additional plot parameters
+                plt.title("Sent packets per connection", y=title_distance)
+                plt.xlabel('Number of Packets')
+                plt.ylabel('Connection')
+                width = 0.5
+                plt.grid(True)
+                plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot 
+                plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
+
+                # plot the above data, first use plain numbers as graphy to maintain sorting
+                plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+                # now change the y numbers to the respective address labels
+                plt.yticks(range(len(graphy)), graphy)
+                # use tight layout to cut off unnecessary space
+                plt.tight_layout(pad=4)
+
+                # save created figure
+                out = self.pcap_filepath.replace('.pcap', '_plot-connection' + file_ending)
+                plt.savefig(out, dpi=500)
+                return out
+            else:
+                print("Error plot protocol: No protocol values found!")
+
+        def plot_out_degree(file_ending: str):
+            plt.gcf().clear()
+            out_degree = self.get_out_degree()
+            #print("")
+            #print("#############in plot_out_degree###########")
+            #print(out_degree)
+
+            graphx, graphy = [], []
+            for entry in out_degree:
+                graphx.append(entry[0])
+                graphy.append(entry[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Outdegree")
+            plt.xlabel('IpAddress')
+            plt.ylabel('Outdegree')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            x = range(0,len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_out_degree' + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+        def plot_in_degree(file_ending: str):
+            plt.gcf().clear()
+            in_degree = self.get_in_degree()
+
+            graphx, graphy = [], []
+            for entry in in_degree:
+                graphx.append(entry[0])
+                graphy.append(entry[1])
+            plt.autoscale(enable=True, axis='both')
+            plt.title("Indegree")
+            plt.xlabel('IpAddress')
+            plt.ylabel('Indegree')
+            width = 0.1
+            plt.xlim([0, len(graphx)])
+            plt.grid(True)
+
+            x = range(0,len(graphx))
+            my_xticks = graphx
+            plt.xticks(x, my_xticks)
+
+            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
+            out = self.pcap_filepath.replace('.pcap', '_in_degree' + file_ending)
+            plt.savefig(out,dpi=500)
+            return out
+
+
         ttl_out_path = plot_ttl('.' + format)
         mss_out_path = plot_mss('.' + format)
         win_out_path = plot_win('.' + format)
@@ -940,6 +1150,9 @@ class Statistics:
         plot_interval_new_tos = plot_interval_new_tos('.' + format)
         plot_interval_new_win_size = plot_interval_new_win_size('.' + format)
         plot_interval_new_mss = plot_interval_new_mss('.' + format)
+        plot_packets_per_connection_out = plot_packets_per_connection('.' + format)
+        plot_out_degree = plot_out_degree('.' + format)
+        plot_in_degree = plot_in_degree('.' + format)
 
         ## Time consuming plot
         # port_out_path = plot_port('.' + format)
@@ -948,3 +1161,4 @@ class Statistics:
         # ip_dst_out_path = plot_ip_dst('.' + format)
 
         print("Saved plots in the input PCAP directory.")
+        print("In-/Out-/Overall-degree plots not fully finished yet")