Add local IP recognition in local networks with public IPs.

Fix IPv4 address pattern. Make messages a dict.

code/Attack/MembersMgmtCommAttack.py

@@ -353,8 +353,8 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         # assign addresses for local IDs
         if number_local_ids > 0:
             reuse_count_local = int(reuse_percent_total * reuse_percent_local * number_local_ids) 
-            existing_local_ips = sorted(pcapops.get_existing_priv_ips(reuse_count_local))
-            new_local_ips = sorted(pcapops.get_new_priv_ips(number_local_ids - len(existing_local_ips)))
+            existing_local_ips = sorted(pcapops.get_existing_local_ips(reuse_count_local))
+            new_local_ips = sorted(pcapops.get_new_local_ips(number_local_ids - len(existing_local_ips)))
             add_ids_to_config(sorted(local_ids), existing_local_ips, new_local_ips, bot_configs)
 
         # assign addresses for external IDs
@@ -381,7 +381,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         # configurations (i.e. IP, MAC, port, ...) for easier later use
         final_messages = []
         new_id = 0
-        for msg in messages:
+        for msg in sorted(messages.values(), key=lambda msg: msg.msg_id):
             type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]
             id_src, id_dst = msg.src, msg.dst
 
@@ -400,12 +400,12 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
 
 
     def _get_capture_duration(self):
+        """
+        Returns the duration of the input PCAP (since statistics duration seems to be incorrect)
+        """
         ts_date_format = "%Y-%m-%d %H:%M:%S.%f"
         ts_first_date = datetime.strptime(self.statistics.get_pcap_timestamp_start(), ts_date_format)
         ts_last_date = datetime.strptime(self.statistics.get_pcap_timestamp_end(), ts_date_format)
         diff_date = ts_last_date - ts_first_date
         duration = "%d.%d" % (diff_date.total_seconds(), diff_date.microseconds)
-        return duration
-
-
-
+        return duration

code/ID2TLib/CommunicationProcessor.py

@@ -162,14 +162,14 @@ class CommunicationProcessor():
         On the side also connect corresponding messages together to quickly find out
         which reply belongs to which request and vice versa.
 
-        :return: a 4-tuple as (initiator IDs, responder IDs, both IDs, messages)
+        :return: a triple as (initiator IDs, responder IDs, messages)
         """
 
         mtypes = self.mtypes
         # setup initial variables and their values
         respnd_ids = set()
         # msgs --> the filtered messages, msg_id --> an increasing ID to give every message an artificial primary key
-        msgs, msg_id = [], 0
+        msgs, msg_id = {}, 0
         # keep track of previous request to find connections
         prev_reqs = {}
         init_ids = self.init_ids
@@ -193,7 +193,7 @@ class CommunicationProcessor():
                 # convert the abstract message into a message object to handle it better
                 msg_str = "{0}-{1}".format(id_src, id_dst)
                 msg = Message(msg_id, id_src, id_dst, msg_type, time)
-                msgs.append(msg)
+                msgs[msg_id] = msg
                 prev_reqs[msg_str] = msg_id
 
             # process a reply
@@ -209,7 +209,7 @@ class CommunicationProcessor():
                 msgs[refer_idx].refer_msg_id = msg_id
                 # print(msgs[refer_idx])
                 msg = Message(msg_id, id_src, id_dst, msg_type, time, refer_idx)
-                msgs.append(msg)
+                msgs[msg_id] = msg
                 # remove the request to this response from storage
                 del(prev_reqs[msg_str])
 

code/ID2TLib/IPv4.py

@@ -2,7 +2,7 @@ import re
 
 class IPAddress:
 	# a number between 0 and 255, no leading zeros
-	_IP_NUMBER_REGEXP = r"(25[0-5]|2[0-4]\d|1?[1-9]?\d)"
+	_IP_NUMBER_REGEXP = r"(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)"
 	# 4 numbers between 0 and 255, joined together with dots
 	IP_REGEXP = r"{0}\.{0}\.{0}\.{0}".format(_IP_NUMBER_REGEXP)
 	

code/ID2TLib/PcapAddressOperations.py

@@ -31,226 +31,192 @@ class PcapAddressOperations():
         """
         return self.contains_priv_ips
 
-    def get_priv_address_range(self):
+    def get_local_address_range(self):
         """
-        Returns a tuple with the start and end of the observed private IP range.
+        Returns a tuple with the start and end of the observed local IP range.
         """
-        # better way to handle error?
-        if not self.pcap_contains_priv_ips():
-            print("Error: .pcap does not contain any private ips.")
-            return -1, -1
-        return str(self.min_priv_ip), str(self.max_priv_ip)
+        return str(self.min_local_ip), str(self.max_local_ip)
 
-    def get_count_rem_priv_ips(self):
+    def get_count_rem_local_ips(self):
         """
-        Returns the number of private IPs in the pcap file that have not aldready been returned by get_existing_priv_ips.
+        Returns the number of local IPs in the pcap file that have not aldready been returned by get_existing_local_ips.
         """
-        return len(self.remaining_priv_ips)
+        return len(self.remaining_local_ips)
 
-    def get_existing_priv_ips(self, count: int=1):
+    def get_existing_local_ips(self, count: int=1):
         """
-        Returns the given number of private IPs that are existent in the pcap file.
+        Returns the given number of local IPs that are existent in the pcap file.
 
-        :param count: the number of IPs to return
-        :return: the chosen private IPs
+        :param count: the number of local IPs to return
+        :return: the chosen local IPs
         """
 
-        # reasonable to include this?
-        if not self.pcap_contains_priv_ips():
-            print("Warning: .pcap does not contain any private ips.")
-            return []
+        if count > len(self.remaining_local_ips):
+            print("Warning: There are no more {} local IPs in the .pcap file. Returning all remaining local IPs.".format(count))
 
-        if count > len(self.remaining_priv_ips):
-            print("Warning: There are no more {} private IPs in the .pcap file. Returning all remaining private IPs.".format(count))
+        total = min(len(self.remaining_local_ips), count)
 
-        total = min(len(self.remaining_priv_ips), count)
-
-        retr_priv_ips = []
-        priv_ips = self.remaining_priv_ips
+        retr_local_ips = []
+        local_ips = self.remaining_local_ips
         for _ in range(0, total):
-            random_priv_ip = choice(tuple(priv_ips))
-            retr_priv_ips.append(str(random_priv_ip))
-            priv_ips.remove(random_priv_ip)
+            random_local_ip = choice(sorted(local_ips))
+            retr_local_ips.append(str(random_local_ip))
+            local_ips.remove(random_local_ip)
 
         # if count == 1:
-        #     return retr_priv_ips[0]
+        #     return retr_local_ips[0]
 
-        return retr_priv_ips
+        return retr_local_ips
 
     # also use IPs below minimum observed IP?
     # offset for later, start at x after minimum? e.g. start at 192.168.0.100
     # exclude the last IP of an IP segment because its broadcast?
-    def get_new_priv_ips(self, count: int=1):
+    def get_new_local_ips(self, count: int=1):
         """
-        Returns in the pcap not existent private IPs that match the used segment. IPs can be returned
+        Returns in the pcap not existent local IPs that are in proximity of the observed local IPs. IPs can be returned
         that are either between the minimum and maximum observed IP and are therefore considered certain
-        or that are above the observed maximum address, are more likely to not belong to the network using the
-        private IP segment and are therefore considered uncertain.
+        or that are above the observed maximum address, are more likely to not belong to the local network 
+        and are therefore considered uncertain.
 
-        :param count: the number of IPs to return
-        :return: the newly created private IP addresses
+        :param count: the number of new local IPs to return
+        :return: the newly created local IP addresses
         """
 
-        if not self.pcap_contains_priv_ips():
-            print("Error: .pcap does not contain any private ips.")
-            return []
-
-        unused_priv_ips = self.unused_priv_ips
-        uncertain_priv_ips = self.uncertain_priv_ips
+        unused_local_ips = self.unused_local_ips
+        uncertain_local_ips = self.uncertain_local_ips
 
         # warning reasonable?
-        if count > len(unused_priv_ips):
-            print("Warning: there are no {0} unused certain priv IPs in the .pcap file.\n \
-                Returning {1} certain and {2} uncertain priv IPs.".format(count, len(unused_priv_ips), count-len(unused_priv_ips)))
+        if count > len(unused_local_ips):
+            print("Warning: there are no {0} unused certain local IPs in the .pcap file.\n \
+                Returning {1} certain and {2} uncertain local IPs.".format(count, len(unused_local_ips), count-len(unused_local_ips)))
 
-        count_certain = min(count, len(unused_priv_ips))
+        count_certain = min(count, len(unused_local_ips))
     
-        retr_priv_ips = []
+        retr_local_ips = []
         for _ in range(0, count_certain):
-            random_priv_ip = choice(tuple(unused_priv_ips))
-            retr_priv_ips.append(str(random_priv_ip))
-            unused_priv_ips.remove(random_priv_ip)
+            random_local_ip = choice(sorted(unused_local_ips))
+            retr_local_ips.append(str(random_local_ip))
+            unused_local_ips.remove(random_local_ip)
 
-        # retrieve uncertain priv ips
+        # retrieve uncertain local ips
         if count_certain < count:
             count_uncertain = count - count_certain
 
             # check if new uncertain IPs have to be created
-            if len(uncertain_priv_ips) < count_uncertain:
+            if len(uncertain_local_ips) < count_uncertain:
                 ipspace_multiplier = self.UNCERTAIN_IPSPACE_MULTIPLIER
 
-                max_new_ip = self.max_uncertain_priv_ip.to_int() + ipspace_multiplier * count_uncertain
-                # adjust IP space multiplier and prevent broadcast address of private segment from being chosen as new IP
-                while max_new_ip >= self.priv_ip_segment.last_address().to_int():
-                    max_new_ip -= 1
-            
-                count_new_ips = max_new_ip - self.max_uncertain_priv_ip.to_int()
-                if count_new_ips < count_uncertain:
-                    print("Error: Cannot generate enough new private IPs because they would exceed the maximum private segment IP. Returning {}.".format(count_new_ips))
+                max_new_ip = self.max_uncertain_local_ip.to_int() + ipspace_multiplier * count_uncertain
 
-                # create ipspace_multiplier * count_uncertain new uncertain IP addresses
+                count_new_ips = max_new_ip - self.max_uncertain_local_ip.to_int()
+
+                # create ipspace_multiplier * count_uncertain new uncertain local IP addresses
                 last_gen_ip = None
                 for i in range(1, count_new_ips + 1):
-                    ip = IPAddress.from_int(self.max_uncertain_priv_ip.to_int() + i)
-                    # exclude the broadcast address
-                    if ip.to_int() >= self.priv_ip_segment.last_address().to_int():
-                        break
-                    uncertain_priv_ips.add(ip)
+                    ip = IPAddress.from_int(self.max_uncertain_local_ip.to_int() + i)
+                    # exclude the definite broadcast address
+                    if self.priv_ip_segment:
+                        if ip.to_int() >= self.priv_ip_segment.last_address().to_int():
+                            break
+                    uncertain_local_ips.add(ip)
                     last_gen_ip = ip
-                self.max_uncertain_priv_ip = last_gen_ip
+                self.max_uncertain_local_ip = last_gen_ip
 
             # choose the uncertain IPs to return
-            total_uncertain = min(count_uncertain, len(uncertain_priv_ips))
+            total_uncertain = min(count_uncertain, len(uncertain_local_ips))
             for _ in range(0, total_uncertain):
-                random_priv_ip = choice(tuple(uncertain_priv_ips))
-                retr_priv_ips.append(str(random_priv_ip))
-                uncertain_priv_ips.remove(random_priv_ip)
+                random_local_ip = choice(sorted(uncertain_local_ips))
+                retr_local_ips.append(str(random_local_ip))
+                uncertain_local_ips.remove(random_local_ip)
 
         # if count == 1:
-        #     return retr_priv_ips[0]
+        #     return retr_local_ips[0]
             
-        return retr_priv_ips
+        return retr_local_ips
 
     def get_existing_external_ips(self, count: int=1):
         """
-        Returns the given number of private IPs that are existent in the pcap file.
+        Returns the given number of external IPs that are existent in the pcap file.
 
-        :param count: the number of IPs to return
-        :return: the chosen private IPs
+        :param count: the number of external IPs to return
+        :return: the chosen external IPs
         """
 
         # reasonable to include this?
-        if not (len(self.public_ips) > 0):
-            print("Warning: .pcap does not contain any public ips.")
+        if not (len(self.external_ips) > 0):
+            print("Warning: .pcap does not contain any external ips.")
             return []
 
-        if count > len(self.remaining_public_ips):
-            print("Warning: There are no {} public IPs in the .pcap file. Returning all existing public IPs.".format(count))
+        if count > len(self.remaining_external_ips):
+            print("Warning: There are no more %d external IPs in the .pcap file.\n" % count +
+                "Returning all %d existing external IPs." % len(self.remaining_external_ips))
 
-        total = min(len(self.remaining_public_ips), count)
+        total = min(len(self.remaining_external_ips), count)
 
-        retr_public_ips = []
-        public_ips = self.remaining_public_ips
+        retr_external_ips = []
+        external_ips = self.remaining_external_ips
 
         for _ in range(0, total):
-            random_public_ip = choice(tuple(public_ips))
-            retr_public_ips.append(str(random_public_ip))
-            public_ips.remove(random_public_ip)
+            random_external_ip = choice(sorted(external_ips))
+            retr_external_ips.append(str(random_external_ip))
+            external_ips.remove(random_external_ip)
 
         # if count == 1:
-        #     return retr_public_ips[0]
+        #     return retr_external_ips[0]
 
-        return retr_public_ips
+        return retr_external_ips
 
     def _init_ipaddress_ops(self):
         """
         Load and process data needed to perform functions on the IP addresses contained in the statistics
         """
 
-        all_ips = self.statistics.process_db_query("all(ipAddress)", print_results=False)
-
-        # Prepare private address operations and on the side save all external IPs
-        # find the private IP segment in use
-        public_ips = set()
-        priv_ip_segment = None
+        # retrieve local and external IPs
+        all_ips_str = set(self.statistics.process_db_query("all(ipAddress)", print_results=False))
+        external_ips_str = set(self.statistics.process_db_query("ipAddress(macAddress=%s)" % self.get_probable_router_mac(), print_results=False))
+        local_ips_str = all_ips_str - external_ips_str
+        external_ips = set()
+        local_ips = set()
         self.contains_priv_ips = False
-        first_priv_ip = None
-        first_priv_ip_idx = -1
-
-        # for that iterate over all IPs until the first private IP is found
-        for i, ip in enumerate(all_ips):
-            if not is_ipv4(ip): 
-                continue
-            ip = IPAddress.parse(ip)
-
-            if ip.is_private():
-                priv_ip_segment = ip.get_private_segment()
-                first_priv_ip_idx = i
-                first_priv_ip = ip
-                self.contains_priv_ips = True
-                break
-            # new function in IPv4 to shorten this?
-            elif (not ip.is_localhost()) and (not ip.is_localhost()) and (not ip.is_multicast()) and (not ip.is_reserved()) and (not ip.is_zero_conf()):
-                public_ips.add(ip)
-
-        if not self.contains_priv_ips:
-            #print("The Pcap File does not contain any private IPs")
-            return
-
-        # get minimum and maximum seen private IP. The addresses in-bewteen are considered 
-        # as certain to be part of the network the pcap traffic is from
-        min_priv_ip, max_priv_ip = first_priv_ip, first_priv_ip
-        priv_ips = {first_priv_ip}
-        for ip in all_ips[first_priv_ip_idx+1:]:
-            if not is_ipv4(ip): 
-                continue
-            ip = IPAddress.parse(ip)
-
-            if ip in priv_ip_segment:
-                priv_ips.add(ip)
-                if ip > max_priv_ip:
-                    max_priv_ip = ip
-                elif ip < min_priv_ip:
-                    min_priv_ip = ip
-            # new function in IPv4 to shorten this?
-            elif (not ip.is_private()) and (not ip.is_localhost()) and (not ip.is_localhost()) and (not ip.is_multicast()) and (not ip.is_reserved()) and (not ip.is_zero_conf()):
-                public_ips.add(ip)
-
-        # save the certain unused priv IPs of the network
-        unused_priv_ips = set()
-        for i in range (min_priv_ip.to_int() + 1, max_priv_ip.to_int()):
+        self.priv_ip_segment = None
+
+        # convert local IP strings to IPv4.IPAddress representation
+        for ip in local_ips_str:
+            if is_ipv4(ip):
+                ip = IPAddress.parse(ip)
+                if ip.is_private() and not self.contains_priv_ips:
+                    self.contains_priv_ips = True
+                    self.priv_ip_segment = ip.get_private_segment()
+                if (not str(ip) == "255.255.255.255") and (not ip.is_localhost()) and (not ip.is_multicast()) and (not ip.is_reserved()) and (not ip.is_zero_conf()):
+                    local_ips.add(ip)
+
+        # convert external IP strings to IPv4.IPAddress representation
+        for ip in external_ips_str:
+            if is_ipv4(ip):
+                ip = IPAddress.parse(ip)
+                # if router MAC can definitely be mapped to local/private IP, add it to local_ips
+                if ip.is_private():
+                    local_ips.add(ip)
+                # new function in IPv4 to shorten this?
+                # exclude local broadcast address
+                elif (not str(ip) == "255.255.255.255") and (not ip.is_localhost()) and (not ip.is_multicast()) and (not ip.is_reserved()) and (not ip.is_zero_conf()):
+                    external_ips.add(ip)
+
+        min_local_ip, max_local_ip = min(local_ips), max(local_ips)
+
+        # save the certain unused local IPs of the network
+        unused_local_ips = set()
+        for i in range(min_local_ip.to_int() + 1, max_local_ip.to_int()):
             ip = IPAddress.from_int(i)
-            if not ip in priv_ips:
-                unused_priv_ips.add(ip)
+            if not ip in local_ips:
+                unused_local_ips.add(ip)
 
         # save the gathered information for efficient later use
-        self.public_ips = frozenset(public_ips)
-        self.remaining_public_ips = public_ips
-        self.min_priv_ip, self.max_priv_ip = min_priv_ip, max_priv_ip
-        self.max_uncertain_priv_ip = max_priv_ip
-        self.priv_ips = frozenset(priv_ips)
-        self.remaining_priv_ips = priv_ips
-        self.unused_priv_ips = unused_priv_ips
-        self.generated_uncertain_ips = set()
-        self.uncertain_priv_ips = set()
-        self.priv_ip_segment = priv_ip_segment
+        self.external_ips = frozenset(external_ips)
+        self.remaining_external_ips = external_ips
+        self.min_local_ip, self.max_local_ip = min_local_ip, max_local_ip
+        self.max_uncertain_local_ip = max_local_ip
+        self.local_ips = frozenset(local_ips)
+        self.remaining_local_ips = local_ips
+        self.unused_local_ips = unused_local_ips
+        self.uncertain_local_ips = set()