Merge branch 'develop' into merge_lib

code/Attack/MembersMgmtCommAttack.py

@@ -187,7 +187,10 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             id_src, id_dst = msg.src["ID"], msg.dst["ID"]
             ip_src, ip_dst = msg.src["IP"], msg.dst["IP"]
             mac_src, mac_dst = msg.src["MAC"], msg.dst["MAC"]
-            port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
+            if msg.type.is_request():
+                port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
+            else:
+                port_src, port_dst = int(msg.src["DstPort"]), int(msg.dst["SrcPort"])
             ttl = int(msg.src["TTL"])
 
             # update duration

code/ID2TLib/Statistics.py

@@ -560,14 +560,14 @@ class Statistics:
 
     def get_in_degree(self):
         """
-        determines the in-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has received packets from
-        :return: a list, each entry consists of one local IPAddress and its associated in-degree
+        determines the in-degree for each ipAddress, i.e. for every IP the count of ipAddresses it has received packets from
+        :return: a list, each entry consists of one IPAddress and its associated in-degree
         """
 
         in_degree_raw = self.stats_db._process_user_defined_query(
-                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'in\' AND portNumber = portA GROUP BY ipAddress " +
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressA WHERE portDirection=\'in\' AND portNumber = portA GROUP BY ipAddress " +
                 "UNION " +
-                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'in\' AND portNumber = portB GROUP BY ipAddress")
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressB WHERE portDirection=\'in\' AND portNumber = portB GROUP BY ipAddress")
 
         #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
         in_degree = self.filter_multiples(in_degree_raw)
@@ -576,61 +576,49 @@ class Statistics:
 
     def get_out_degree(self):
         """
-        determines the out-degree for each local ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
-        :return: a list, each entry consists of one local IPAddress and its associated out-degree
-        """
-        """
-
-        test = self.stats_db._process_user_defined_query("SELECT DISTINCT * FROM conv_statistics")
-        #test2 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddressB, portB FROM conv_statistics")
-        print("############# conv_statistics IP's + Ports")
-        for p in test:
-            print(p)
-        #for p in test2:
-        #    print(p)
-
-        print("############## ip_ports ##################")
-        test3 = self.stats_db._process_user_defined_query("SELECT DISTINCT ipAddress, portNumber, portDirection FROM ip_ports")
-        for p in test3:
-            print(p)
-
-        print("")
-        print("############## AFTER JOIN - A #############")
-        test4 = self.stats_db._process_user_defined_query(
-                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA") # Hier werden die anfang locals rausgefiltert!
-        for p in test4:
-            print(p)
-
-        print("")
-        print("############## AFTER JOIN - B #############")
-        test6 = self.stats_db._process_user_defined_query(
-                "SELECT * FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB") # Hier werden die anfang locals rausgefiltert!
-        for p in test6:
-            print(p)
-
-        print("")
-        print("############## BUILD UP PART FOR PART#############")
-        test5 = self.stats_db._process_user_defined_query(
-                "SELECT ipAddress, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' GROUP BY ipAddress")
-        for p in test5:
-            print(p)
+        determines the out-degree for each ipAddress, i.e. for every IP the count of ipAddresses it has sent packets to
+        :return: a list, each entry consists of one IPAddress and its associated out-degree
         """
+        
         out_degree_raw = self.stats_db._process_user_defined_query(
-                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA GROUP BY ipAddress " +
+                "SELECT ipAddressA, Count(DISTINCT ipAddressB) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressA WHERE portDirection=\'out\' AND portNumber = portA GROUP BY ipAddress " +
                 "UNION " +
-                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB GROUP BY ipAddress")
-
-        #filter out non-local IPs
-        #out_degree_raw_2 = []
-        #for entry in out_degree_raw:
-        #    if IPAddress.parse(entry[0]).is_reserved():
-        #        out_degree_raw_2.append(entry)
+                "SELECT ipAddressB, Count(DISTINCT ipAddressA) FROM ip_ports JOIN conv_statistics_stateless ON ipAddress = ipAddressB WHERE portDirection=\'out\' AND portNumber = portB GROUP BY ipAddress")
 
         #Because of the structure of the database, there could be 2 entries for the same IP Address, therefore accumulate their sums
         out_degree = self.filter_multiples(out_degree_raw)
 
         return out_degree
 
+    def filter_multiples(self, entries):
+        """
+        helper function, for get_out_degree and get_in_degree
+        filters the given list for duplicate IpAddresses and, if duplciates are present, accumulates their values
+
+        :param entries: list, each entry consists of an ipAddress and a numeric value
+        :return: a filtered list, without duplicate ipAddresses
+        """
+
+        filtered_entries = []
+        done = []
+        for p1 in entries:
+            added = False
+            if p1 in done:
+                continue
+            for p2 in entries:
+                if p1[0] == p2[0] and p1 != p2:
+                    filtered_entries.append((p1[0], p1[1] + p2[1]))
+                    done.append(p1)
+                    done.append(p2)
+                    print("duplicate found:", p1, " and ", p2)
+                    added = True
+                    break
+
+            if not added:
+                filtered_entries.append(p1)
+
+        return filtered_entries
+
     def get_avg_delay_local_ext(self):
         """
         Calculates the average delay of a packet for external and local communication, based on the tcp handshakes
@@ -677,36 +665,6 @@ class Statistics:
             avg_delay_local = 0.06
         return avg_delay_local, avg_delay_external
 
-    def filter_multiples(self, entries):
-        """
-        helper function, for get_out_degree and get_in_degree
-        filters the given list for duplicate IpAddresses and, if duplciates are present, accumulates their values
-
-        :param entries: list, each entry consists of an ipAddress and a numeric value
-        :return: a filtered list, without duplicate ipAddresses
-        """
-
-        filtered_entries = []
-        done = []
-        for p1 in entries:
-            added = False
-            if p1 in done:
-                continue
-            for p2 in entries:
-                if p1[0] == p2[0] and p1 != p2:
-                    filtered_entries.append((p1[0], p1[1] + p2[1]))
-                    done.append(p1)
-                    done.append(p2)
-                    #entries.remove(p2)
-                    added = True
-                    break
-
-            if not added:
-                filtered_entries.append(p1)
-
-        return filtered_entries
-
-
     def get_statistics_database(self):
         """
         :return: A reference to the statistics database object
@@ -1087,136 +1045,134 @@ class Statistics:
                 plt.savefig(out, dpi=500)
                 return out
 
-        def plot_packets_per_connection(file_ending: str):
+        def plot_in_degree(file_ending: str):
             """
-            Plots the exchanged packets per connection as horizontal bar plot. 
-            Included are 'half-open' connections, where only one packet is exchanged.
-            Note: there may be cutoff problems within the plot if there is to little data.
+            Creates a Plot, visualizing the in-degree for every IP Address
 
-            :param file_ending: The file extension for the output file containing the plot
+            :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
             :return: A filepath to the file containing the created plot
             """
+
             plt.gcf().clear()
-            result = self.stats_db._process_user_defined_query(
-                "SELECT ipAddressA, portA, ipAddressB, portB, pktsCount FROM conv_statistics_stateless")
 
-            if (result):
-                graphy, graphx = [], []
-                # plot data in descending order
-                result = sorted(result, key=lambda row: row[4])
+            # retrieve data
+            in_degree = self.get_in_degree()
 
-                # compute plot data
-                for i, row in enumerate(result):
-                    addr1, addr2 = "%s:%d" % (row[0], row[1]), "%s:%d" % (row[2], row[3])
-                    # adjust the justification of strings to improve appearance
-                    len_max = max(len(addr1), len(addr2))
-                    addr1 = addr1.ljust(len_max)
-                    addr2 = addr2.ljust(len_max)
-                    # add plot data
-                    graphy.append("%s\n%s" % (addr1, addr2))
-                    graphx.append(row[4])
+            if(in_degree):
+                graphx, graphy = [], []
+                for entry in in_degree:
+                    # degree values
+                    graphx.append(entry[1])
+                    # IP labels
+                    graphy.append(entry[0])
 
-                # compute plot height in inches
-                dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
-                plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
-                title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
-
-                # have x axis and its label appear at the top (instead of bottom)
-                fig, ax = plt.subplots()
-                ax.xaxis.tick_top()
-                ax.xaxis.set_label_position("top")
-
-                # set additional plot parameters
-                plt.title("Sent packets per connection", y=title_distance)
-                plt.xlabel('Number of Packets')
-                plt.ylabel('Connection')
-                width = 0.5
+                # set labels
+                plt.title("Indegree per IP Address")
+                plt.ylabel('IpAddress')
+                plt.xlabel('Indegree')
+
+                #set width of the bars
+                width = 0.3
+
+                # set scalings
+                plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5))  # these proportions just worked well
+
+                #set limits of the axis
+                plt.ylim([0, len(graphy)])
+                plt.xlim([0, max(graphx) + 10])
+
+                # display numbers at each bar
+                for i, v in enumerate(graphx):
+                    plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
+
+                # display grid for better visuals
                 plt.grid(True)
-                plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot
-                plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
-
-                # plot the above data, first use plain numbers as graphy to maintain sorting
-                plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
-                # now change the y numbers to the respective address labels
-                plt.yticks(range(len(graphy)), graphy)
-                # try to use tight layout to cut off unnecessary space
-                try:
-                    plt.tight_layout(pad=4)
-                except ValueError:
-                    pass
 
-                # save created figure
-                out = self.pcap_filepath.replace('.pcap', '_plot-PktCount per Connection Distribution' + file_ending)
-                plt.savefig(out, dpi=500)
+                # plot the bar
+                labels = graphy
+                graphy = list(range(len(graphx)))
+                plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+                plt.yticks(graphy, labels)
+                out = self.pcap_filepath.replace('.pcap', '_in_degree' + file_ending)
+                plt.tight_layout()
+                plt.savefig(out,dpi=500)
                 return out
             else:
-                print("Error plot protocol: No protocol values found!")
+                print("Error: No statistics Information for plotting out-degrees found")
 
         def plot_out_degree(file_ending: str):
+            """
+            Creates a Plot, visualizing the out-degree for every IP Address
+
+            :param file_ending: The file extension for the output file containing the plot, e.g. "pdf"
+            :return: A filepath to the file containing the created plot
+            """
+
             plt.gcf().clear()
+
+            # retrieve data
             out_degree = self.get_out_degree()
-            #print("")
-            #print("#############in plot_out_degree###########")
-            #print(out_degree)
 
-            graphx, graphy = [], []
-            for entry in out_degree:
-                graphx.append(entry[0])
-                graphy.append(entry[1])
-            plt.autoscale(enable=True, axis='both')
-            plt.title("Outdegree")
-            plt.xlabel('IpAddress')
-            plt.ylabel('Outdegree')
-            width = 0.1
-            plt.xlim([0, len(graphx)])
-            plt.grid(True)
+            if(out_degree):
+                graphx, graphy = [], []
+                for entry in out_degree:
+                    # degree values
+                    graphx.append(entry[1])
+                    # IP labels
+                    graphy.append(entry[0])
 
-            x = range(0,len(graphx))
-            my_xticks = graphx
-            plt.xticks(x, my_xticks)
+                # set labels
+                plt.title("Outdegree per IP Address")
+                plt.ylabel('IpAddress')
+                plt.xlabel('Outdegree')
 
-            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
-            out = self.pcap_filepath.replace('.pcap', '_out_degree' + file_ending)
-            plt.savefig(out,dpi=500)
-            return out
+                #set width of the bars
+                width = 0.3
 
-        def plot_in_degree(file_ending: str):
-            plt.gcf().clear()
-            in_degree = self.get_in_degree()
+                # set scalings
+                plt.figure(figsize=(int(len(graphx))/20 + 5, int(len(graphy)/5) + 5))  # these proportions just worked well
 
-            graphx, graphy = [], []
-            for entry in in_degree:
-                graphx.append(entry[0])
-                graphy.append(entry[1])
-            plt.autoscale(enable=True, axis='both')
-            plt.title("Indegree")
-            plt.xlabel('IpAddress')
-            plt.ylabel('Indegree')
-            width = 0.1
-            plt.xlim([0, len(graphx)])
-            plt.grid(True)
+                #set limits of the axis
+                plt.ylim([0, len(graphy)])
+                plt.xlim([0, max(graphx) + 10])
 
-            x = range(0,len(graphx))
-            my_xticks = graphx
-            plt.xticks(x, my_xticks)
+                # display numbers at each bar
+                for i, v in enumerate(graphx):
+                    plt.text(v + 1, i + .1, str(v), color='blue', fontweight='bold')
 
-            plt.bar(x, graphy, width, align='center', linewidth=1, color='red', edgecolor='red')
-            out = self.pcap_filepath.replace('.pcap', '_in_degree' + file_ending)
-            plt.savefig(out,dpi=500)
-            return out
+                # display grid for better visuals
+                plt.grid(True)
+
+                # plot the bar
+                labels = graphy
+                graphy = list(range(len(graphx)))
+                plt.barh(graphy, graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+                plt.yticks(graphy, labels)
+                out = self.pcap_filepath.replace('.pcap', '_out_degree' + file_ending)
+                plt.tight_layout()
+                plt.savefig(out,dpi=500)
+                return out
+            else:
+                print("Error: No statistics Information for plotting out-degrees found")
 
-        def plot_avgpkts_per_comm_interval(file_ending: str):
+        def plot_big_comm_interval_stat(attr:str, table:str, title:str, xlabel:str, suffix:str):
             """
-            Plots the exchanged packets per connection as horizontal bar plot. 
+            Plots the desired statistc per connection as horizontal bar plot. 
             Included are 'half-open' connections, where only one packet is exchanged.
-            Note: there may be cutoff problems within the plot if there is to little data.
-
-            :param file_ending: The file extension for the output file containing the plot
+            The given statistics table has to have at least the attributes 'ipAddressA', 'portA', 'ipAddressB',
+            'portB' and the specified additional attribute.
+            Note: there may be cutoff/scaling problems within the plot if there is too little data.
+
+            :param attr: The desired statistic, named with respect to its attribute in the given statistics table
+            :param table: The statistics table 
+            :param title: The title of the created plot
+            :param xlabel: The name of the x-axis of the created plot
+            :param suffix: The suffix of the created file, including file extension
             :return: A filepath to the file containing the created plot
             """
             plt.gcf().clear()
             result = self.stats_db._process_user_defined_query(
-                "SELECT ipAddressA, portA, ipAddressB, portB, avgPktCount FROM comm_interval_statistics")
+                "SELECT ipAddressA, portA, ipAddressB, portB, %s FROM %s" % (attr, table))
 
             if (result):
                 graphy, graphx = [], []
@@ -1234,41 +1190,81 @@ class Statistics:
                     graphy.append("%s\n%s" % (addr1, addr2))
                     graphx.append(row[4])
 
-                # compute plot height in inches
-                dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
-                plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
-                title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
-
-                # have x axis and its label appear at the top (instead of bottom)
-                fig, ax = plt.subplots()
-                ax.xaxis.tick_top()
-                ax.xaxis.set_label_position("top")
-
-                # set additional plot parameters
-                plt.title("Average number of packets per communication interval", y=title_distance)
-                plt.xlabel('Number of Packets')
-                plt.ylabel('Connection')
-                width = 0.5
-                plt.grid(True)
-                plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot
-                plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
-
-                # plot the above data, first use plain numbers as graphy to maintain sorting
-                plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
-                # now change the y numbers to the respective address labels
-                plt.yticks(range(len(graphy)), graphy)
-                # try to use tight layout to cut off unnecessary space
-                try:
-                    plt.tight_layout(pad=4)
-                except ValueError:
-                    pass
+            # compute plot height in inches
+            dist_mult_height, dist_mult_width = 0.55, 0.07  # these values turned out to work well
+            plt_height, plt_width = len(graphy) * dist_mult_height, max(graphx) * dist_mult_width
+            title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
 
-                # save created figure
-                out = self.pcap_filepath.replace('.pcap', '_plot-Avg PktCount Communication Interval Distribution' + file_ending)
-                plt.savefig(out, dpi=500)
-                return out
-            else:
-                print("Error plot protocol: No protocol values found!")
+            # have x axis and its label appear at the top (instead of bottom)
+            fig, ax = plt.subplots()
+            ax.xaxis.tick_top()
+            ax.xaxis.set_label_position("top")
+
+            # set additional plot parameters
+            plt.title(title, y=title_distance)
+            plt.xlabel(xlabel)
+            plt.ylabel('Connection')
+            width = 0.5
+            plt.grid(True)
+            plt.gca().margins(y=0)  # removes the space between data and x-axis within the plot
+            plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
+
+            # plot the above data, first use plain numbers as graphy to maintain sorting
+            plt.barh(range(len(graphy)), graphx, width, align='center', linewidth=1, color='red', edgecolor='red')
+            # now change the y numbers to the respective address labels
+            plt.yticks(range(len(graphy)), graphy)
+            # try to use tight layout to cut off unnecessary space
+            try:
+                plt.tight_layout(pad=4)
+            except ValueError:
+                pass
+
+            # save created figure
+            out = self.pcap_filepath.replace('.pcap', suffix)
+            plt.savefig(out, dpi=500)
+            return out
+
+        def plot_packets_per_connection(file_ending: str):
+            """
+            Plots the total number of exchanged packets per connection. 
+
+            :param file_ending: The file extension for the output file containing the plot
+            :return: A filepath to the file containing the created plot
+            """
+
+            title = 'Number of exchanged packets per connection'
+            suffix = '_plot-PktCount per Connection Distribution' + file_ending
+
+            # plot data and return outpath
+            return plot_big_comm_interval_stat("pktsCount", "conv_statistics_stateless", title, "Number of packets", suffix)
+
+        def plot_avg_pkts_per_comm_interval(file_ending: str):
+            """
+            Plots the average number of exchanged packets per communication interval for every connection. 
+
+            :param file_ending: The file extension for the output file containing the plot
+            :return: A filepath to the file containing the created plot
+            """
+
+            title = 'Average number of exchanged packets per communication interval'
+            suffix = '_plot-Avg PktCount Communication Interval Distribution' + file_ending
+
+            # plot data and return outpath
+            return plot_big_comm_interval_stat("avgPktCount", "comm_interval_statistics" ,title, "Number of packets", suffix)
+
+        def plot_avg_time_between_comm_interval(file_ending: str):
+            """
+            Plots the average time between the communication intervals of every connection. 
+
+            :param file_ending: The file extension for the output file containing the plot
+            :return: A filepath to the file containing the created plot
+            """
+
+            title = 'Average time between communication intervals in seconds'
+            suffix = '_plot-Avg Time Between Communication Intervals Distribution' + file_ending
+
+            # plot data and return outpath
+            return plot_big_comm_interval_stat("avgTimeBetweenIntervals", "comm_interval_statistics", title, 'Average time between intervals', suffix)
 
 
         ttl_out_path = plot_ttl('.' + format)
@@ -1289,7 +1285,8 @@ class Statistics:
         plot_packets_per_connection_out = plot_packets_per_connection('.' + format)
         plot_out_degree = plot_out_degree('.' + format)
         plot_in_degree = plot_in_degree('.' + format)
-        plot_avgpkts_per_comm_interval_out = plot_avgpkts_per_comm_interval('.' + format)
+        plot_avg_pkts_per_comm_interval_out = plot_avg_pkts_per_comm_interval('.' + format)
+        plot_avg_time_between_comm_interval_out = plot_avg_time_between_comm_interval('.' + format)
 
         ## Time consuming plot
         # port_out_path = plot_port('.' + format)
@@ -1298,4 +1295,3 @@ class Statistics:
         # ip_dst_out_path = plot_ip_dst('.' + format)
 
         print("Saved plots in the input PCAP directory.")
-        print("In-/Out-/Overall-degree plots not fully finished yet")