Merge branch 'develop' of https://git.tk.informatik.tu-darmstadt.de/leon.boeck/ID2T-toolkit-BotnetTraffic into develop

Dockerfile

@@ -1,47 +0,0 @@
-FROM debian:stretch AS build-container
-
-# install all the necessary packages (see readme.md, but libpcap-dev might not be on that list, it's still needed)
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential cmake libboost-dev libboost-python-dev libtins-dev libpcap-dev python3-dev sqlite3 \
-	python3-pip python3-scapy python3-numpy python3-matplotlib python3-scipy python3-setuptools
-RUN pip3 install lea # for some reason you cant install lea via apt
-
-# make the required directored to copy the files into
-RUN mkdir /id2t /id2t/code /id2t/code_boost /id2t/resources
-WORKDIR /id2t
-
-# copy all the necessary files
-# there are multiple commands because docker only copies the directories' contents and not the directory itself and dont know what else to do here
-COPY build.sh /id2t/
-COPY code/ /id2t/code/
-COPY code_boost/ /id2t/code_boost/
-COPY resources /id2t/resources
-
-# run the build-script
-RUN ./build.sh
-
-# use a smaller container for later execution
-# we use python 3.6 because it's the first which comes with stretch
-FROM python:3.6-slim-stretch
-
-# install required libraries
-RUN pip3 install scapy-python3 lea numpy matplotlib scipy
-
-# create and use future work directory, don't copy code_boost
-RUN mkdir /id2t /id2t/code /id2t/resources
-WORKDIR /id2t
-# copy the built project
-COPY --from=build-container /id2t /id2t
-# copy the libs libpcapreader is linked with
-# kinda hacky, but we can do it because both containers share the same os-version
-COPY --from=build-container /usr/lib/x86_64-linux-gnu/libboost_python-py35.so.1.62.0 /usr/lib/libtins.so.3.4 \
-	/usr/lib/x86_64-linux-gnu/libpython3.5m.so.1.0 /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 \
-	/usr/lib/x86_64-linux-gnu/
-
-# install tcpdump because it somehow is a requirement (id2t throws a warning otherwise)
-RUN apt-get update && apt-get install -y tcpdump libpcap0.8 \
-	&& rm -rf /var/lib/apt/lists/*
-
-# add id2t to the path
-ENV PATH="$PATH:/id2t"
-# start with a shell instead of python
-CMD ["bash"]

code/Attack/AttackParameters.py

@@ -55,6 +55,8 @@ class Parameter(Enum):
     PACKET_PADDING = 'packet.padding'
     #recommended type: interval selection strategy, i.e. 'random', 'optimal' or 'custom' ------------------------------------
     INTERVAL_SELECT_STRATEGY = 'interval.selection.strategy'
+    # indicating if the attack will mark generated packets
+    HIDDEN_MARK = "hidden_mark"
 
 class ParameterTypes(Enum):
     """

code/Attack/MembersMgmtCommAttack.py

@@ -69,6 +69,7 @@ from ID2TLib.CommunicationProcessor import CommunicationProcessor
 from ID2TLib.Botnet.MessageMapping import MessageMapping
 from ID2TLib.PcapFile import PcapFile
 from ID2TLib.Statistics import Statistics
+from scapy.layers.inet import IP, IPOption_Security
 
 
 class MembersMgmtCommAttack(BaseAttack.BaseAttack):
@@ -119,7 +120,8 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             # information about the interval selection strategy
             Param.INTERVAL_SELECT_STRATEGY: ParameterTypes.TYPE_INTERVAL_SELECT_STRAT,
             Param.INTERVAL_SELECT_START: ParameterTypes.TYPE_INTEGER_POSITIVE,
-            Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE
+            Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE,
+            Param.HIDDEN_MARK: ParameterTypes.TYPE_BOOLEAN
         }
 
         # create dict with MessageType values for fast name lookup
@@ -171,6 +173,8 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         # interval selection strategy
         self.add_param_value(Param.INTERVAL_SELECT_STRATEGY, "optimal")
 
+        self.add_param_value(Param.HIDDEN_MARK, False)
+
     def generate_attack_pcap(self, context):
         """
         Injects the packets of this attack into a PCAP and stores it as a temporary file.
@@ -196,6 +200,7 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         overThousand = False
 
         msg_packet_mapping = MessageMapping(messages, self.statistics.get_pcap_timestamp_start())
+        mark_packets = self.get_param_value(Param.HIDDEN_MARK)
 
         # create packets to write to PCAP file
         for msg in messages:
@@ -228,6 +233,15 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
             Generator.add_padding(packet, padding,True, True)
 
             packet.time = msg.time
+
+            if mark_packets and isinstance(packet.payload, IP):  # do this only for ip-packets
+                ip_data = packet.payload
+                hidden_opt = IPOption_Security()
+                hidden_opt.option = 2  # "normal" security opt
+                hidden_opt.security = 16  # magic value indicating NSA
+
+                ip_data.options = hidden_opt
+
             packets.append(packet)
             msg_packet_mapping.map_message(msg, packet)
             total_pkts += 1
@@ -605,14 +619,21 @@ class MembersMgmtCommAttack(BaseAttack.BaseAttack):
         assign_realistic_timestamps(messages, external_ids, local_ids, avg_delay_local, avg_delay_external, zero_reference)
 
         portSelector = PortSelectors.LINUX
+        reserved_ports = set(int(line.strip()) for line in open("resources/reserved_ports.txt").readlines())
+        def filter_reserved(get_port):
+            port = get_port()
+            while port in reserved_ports:
+                port = get_port()
+            return port
+
         # create port configurations for the bots
         use_multiple_ports = self.get_param_value(Param.MULTIPORT)
         for bot in sorted(bot_configs):
-            bot_configs[bot]["SrcPort"] = portSelector.select_port_udp()
+            bot_configs[bot]["SrcPort"] = filter_reserved(portSelector.select_port_udp)
             if not use_multiple_ports:
-                bot_configs[bot]["DstPort"] = Generator.gen_random_server_port()
+                bot_configs[bot]["DstPort"] = filter_reserved(Generator.gen_random_server_port)
             else:
-                bot_configs[bot]["DstPort"] = portSelector.select_port_udp()
+                bot_configs[bot]["DstPort"] = filter_reserved(portSelector.select_port_udp)
 
         # assign realistic TTL for every bot
         if self.get_param_value(Param.TTL_FROM_CAIDA):

code/ID2TLib/Statistics.py

@@ -570,6 +570,8 @@ class Statistics:
             # calculate avg local and external delay by summing up the respective delays and dividing them by the number of conversations
             avg_delay_external = 0.0
             avg_delay_local = 0.0
+            default_ext = False
+            default_local = False
 
             if(local_conv):
                 for conv in local_conv:
@@ -577,7 +579,8 @@ class Statistics:
                 avg_delay_local = (avg_delay_local/len(local_conv)) * 0.001 #ms
             else:
                 # no local conversations in statistics found
-                avg_delay_local = 0.06
+                avg_delay_local = 0.055
+                default_local = True
 
             if(external_conv):
                 for conv in external_conv:
@@ -585,11 +588,30 @@ class Statistics:
                 avg_delay_external = (avg_delay_external/len(external_conv)) * 0.001 #ms
             else:
                 # no external conversations in statistics found
-                avg_delay_external = 0.15
+                avg_delay_external = 0.09
+                default_ext = True
         else:
             #if no statistics were found, use these numbers
-            avg_delay_external = 0.15
-            avg_delay_local = 0.06
+            avg_delay_external = 0.09
+            avg_delay_local = 0.055
+            default_ext = True
+            default_local = True
+
+        # check whether delay numbers are consistent
+        if avg_delay_local > avg_delay_external:
+            avg_delay_external = avg_delay_local*1.2
+
+        # print information, that (default) values are used, that are not collected from the Input PCAP
+        if default_ext or default_local:
+            if default_ext and default_local:
+                print("Warning: Could not collect average delays for local or external communication, using following values:")
+            elif default_ext:
+                print("Warning: Could not collect average delays for external communication, using following values:")
+            elif default_local:
+                print("Warning: Could not collect average delays for local communication, using following values:")
+            print("Avg delay of external communication: {0}s,  Avg delay of local communication: {1}s".format(avg_delay_external, avg_delay_local))
+            
+
         return avg_delay_local, avg_delay_external
 
     def get_filtered_degree(self, degree_type: str):
@@ -1211,6 +1233,11 @@ class Statistics:
 
             title_distance = 1 + 0.012*52.8/plt_height  # orginally, a good title distance turned out to be 1.012 with a plot height of 52.8
 
+            # if title would be cut off, set minimum width
+            min_width = len(title) * 0.15
+            if plt_width < min_width:
+                plt_width = min_width
+
             plt.gcf().set_size_inches(plt_width, plt_height)  # set plot size
 
             # set additional plot parameters
@@ -1306,6 +1333,104 @@ class Statistics:
             # plot data and return outpath
             return plot_big_conv_ext_stat("totalConversationDuration", title, 'Duration', suffix)
 
+        def plot_comm_histogram(attr:str, title:str, label:str, suffix:str):
+            """
+            Plots a histogram about the specified attribute for communications.
+            :param attr: The statistics attribute for this histogram
+            :param title: The title of the histogram
+            :param label: The xlabel of the histogram
+            :param suffix: The file suffix
+            :return: The path to the created plot
+            """
+
+            plt.gcf().clear()
+            result_raw = self.stats_db._process_user_defined_query(
+                "SELECT %s FROM conv_statistics_extended" % attr)
+
+            # return without plotting if no data available
+            if not result_raw:
+                return None
+
+            result = []
+            for entry in result_raw:
+                result.append(entry[0])
+
+            # if title would be cut off, set minimum width
+            plt_size = plt.gcf().get_size_inches()
+            min_width = len(title) * 0.12
+            if plt_size[0] < min_width:
+                plt.gcf().set_size_inches(min_width, plt_size[1])  # set plot size
+
+            # set additional plot parameters
+            plt.title(title)
+            plt.ylabel("Number of connections")
+            plt.xlabel(label)
+            width = 0.5
+            plt.grid(True)
+
+            # create 11 bins
+            bins = []
+            max_val = max(result)
+            for i in range(0, 11):
+                bins.append(i * max_val/10)
+
+            # comment out and set weights to normalize
+            # weights = numpy.ones_like(result)/float(len(result))
+
+            # plot the above data, first use plain numbers as graphy to maintain sorting
+            plt.hist(result, bins=bins, color='red', edgecolor='red', align="mid", rwidth=0.5)
+            plt.xticks(bins)
+
+            # save created figure
+            out = self.pcap_filepath.replace('.pcap', suffix)
+            plt.savefig(out, dpi=500)
+            return out
+
+        def plot_histogram_degree(degree_type:str, title:str, label:str, suffix:str):          
+            """
+            Plots a histogram about the specified type for the degree of an IP.
+            :param degree_type: The type of degree, i.e. inDegree, outDegree or overallDegree
+            :param title: The title of the histogram
+            :param label: The xlabel of the histogram
+            :param suffix: The file suffix
+            :return: The path to the created plot
+            """  
+            
+            plt.gcf().clear()
+            result_raw = self.get_filtered_degree(degree_type)
+
+            # return without plotting if no data available
+            if not result_raw:
+                return None
+
+            result = []
+            for entry in result_raw:
+                result.append(entry[1])
+
+            # set additional plot parameters
+            plt.title(title)
+            plt.ylabel("Number of IPs")
+            plt.xlabel(label)
+            width = 0.5
+            plt.grid(True)
+
+            # create 11 bins
+            bins = []
+            max_val = max(result)
+            for i in range(0, 11):
+                bins.append(int(i * max_val/10))
+
+            # comment out and set weights to normalize
+            # weights = numpy.ones_like(result)/float(len(result))
+
+            # plot the above data, first use plain numbers as graphy to maintain sorting
+            plt.hist(result, bins=bins, color='red', edgecolor='red', align="mid", rwidth=0.5)
+            plt.xticks(bins)
+
+            # save created figure
+            out = self.pcap_filepath.replace('.pcap', suffix)
+            plt.savefig(out, dpi=500)
+            return out    
 
         ttl_out_path = plot_ttl('.' + format)
         mss_out_path = plot_mss('.' + format)
@@ -1330,6 +1455,22 @@ class Statistics:
         plot_avg_time_between_comm_interval_out = plot_avg_time_between_comm_interval('.' + format)
         plot_avg_comm_interval_time_out = plot_avg_comm_interval_time("." + format)
         plot_total_comm_duration_out = plot_total_comm_duration("." + format)
+        plot_hist_pkts_per_connection_out = plot_comm_histogram("pktsCount", "Histogram - Number of exchanged packets per connection",
+            "Number of packets", "_plot-Histogram PktCount per Connection" + "." + format)
+        plot_hist_avgpkts_per_commint_out = plot_comm_histogram("avgIntervalPktCount", "Histogram - Average number of exchanged packets per communication interval",
+            "Average number of packets", "_plot-Histogram Avg PktCount per Interval per Connection" + "." + format)
+        plot_hist_avgtime_betw_commints_out = plot_comm_histogram("avgTimeBetweenIntervals", "Histogram - Average time between communication intervals in seconds",
+            "Average time between intervals", "_plot-Histogram Avg Time Between Intervals per Connection" + "." + format)
+        plot_hist_avg_int_time_per_connection_out = plot_comm_histogram("avgIntervalTime", "Histogram - Average duration of a communication interval in seconds",
+            "Average interval time", "_plot-Histogram Avg Interval Time per Connection" + "." + format)
+        plot_hist_total_comm_duration_out = plot_comm_histogram("totalConversationDuration", "Histogram - Total communication duration in seconds",
+            "Duration", "_plot-Histogram Communication Duration per Connection" + "." + format)
+        # plot_hist_indegree_out = plot_histogram_degree("inDegree", "Histogram - Ingoing degree per IP Address", 
+        #     "Ingoing degree", "_plot-Histogram Ingoing Degree per IP" + format)
+        # plot_hist_outdegree_out = plot_histogram_degree("outDegree", "Histogram - Outgoing degree per IP Address", 
+        #     "Outgoing degree", "_plot-Histogram Outgoing Degree per IP" + format)
+        # plot_hist_overalldegree_out = plot_histogram_degree("overallDegree", "Histogram - Overall degree per IP Address", 
+        #     "Overall degree", "_plot-Histogram Overall Degree per IP" + format)
 
         ## Time consuming plot
         # port_out_path = plot_port('.' + format)

resources/reserved_ports.txt

@@ -0,0 +1,483 @@
+1024
+1027
+1027
+1028
+1029
+1058
+1059
+1080
+1085
+1098
+1099
+1109
+1119
+1167
+1194
+1198
+1214
+1220
+1234
+1241
+1270
+1293
+1311
+1341
+1344
+1352
+1360
+1414
+1417
+1418
+1419
+1420
+1431
+1433
+1434
+1512
+1521
+1524
+1527
+1533
+1547
+1581
+1589
+1628
+1629
+1677
+1701
+1701
+1707
+1719
+1720
+1723
+1755
+1783
+1801
+1812
+1813
+1863
+1883
+1900
+1935
+1970
+1972
+1984
+1985
+1998
+2000
+2049
+2080
+2083
+2086
+2095
+2102
+2103
+2104
+2123
+2142
+2152
+2159
+2181
+2210
+2211
+2222
+2223
+2224
+2225
+2226
+2261
+2262
+2266
+2375
+2376
+2379
+2380
+2399
+2401
+2404
+2427
+2447
+2483
+2484
+2535
+2541
+2546
+2547
+2548
+2638
+2727
+2809
+2811
+2944
+2945
+2947
+2948
+2949
+2967
+3020
+3050
+3052
+3074
+3225
+3233
+3260
+3268
+3269
+3283
+3305
+3306
+3386
+3389
+3396
+3412
+3455
+3423
+3424
+3478
+3478
+3478
+3483
+3483
+3493
+3516
+3527
+3544
+3632
+3645
+3659
+3667
+3689
+3690
+3702
+3724
+3725
+3768
+3784
+3799
+3804
+3826
+3830
+3880
+3868
+3872
+3900
+3999
+4018
+4069
+4089
+4090
+4093
+4096
+4105
+4111
+4116
+4172
+4190
+4303
+4307
+4321
+4352
+4486
+4488
+4500
+4502
+4503
+4504
+4505
+4506
+4507
+4508
+4509
+4510
+4511
+4512
+4513
+4514
+4515
+4516
+4517
+4518
+4519
+4520
+4521
+4522
+4523
+4524
+4525
+4526
+4527
+4528
+4529
+4530
+4531
+4532
+4533
+4534
+4569
+4604
+4605
+4662
+4728
+4730
+4739
+4753
+4789
+4840
+4843
+4847
+4894
+4949
+4950
+5000
+5003
+5004
+5005
+5010
+5011
+5048
+5051
+5060
+5061
+5062
+5064
+5065
+5084
+5085
+5093
+5099
+5150
+5151
+5151
+5154
+5172
+5190
+5222
+5228
+5246
+5247
+5269
+5280
+5281
+5298
+5310
+5349
+5349
+5349
+5351
+5353
+5355
+5402
+5405
+5412
+5413
+5417
+5421
+5432
+5556
+5568
+5631
+5632
+5670
+5670
+5672
+5683
+5722
+5741
+5742
+5900
+5931
+5984
+5985
+5986
+5988
+5989
+6000
+6001
+6002
+6003
+6004
+6005
+6006
+6007
+6008
+6009
+6010
+6011
+6012
+6013
+6014
+6015
+6016
+6017
+6018
+6019
+6020
+6021
+6022
+6023
+6024
+6025
+6026
+6027
+6028
+6029
+6030
+6031
+6032
+6033
+6034
+6035
+6036
+6037
+6038
+6039
+6040
+6041
+6042
+6043
+6044
+6045
+6046
+6047
+6048
+6049
+6050
+6051
+6052
+6053
+6054
+6055
+6056
+6057
+6058
+6059
+6060
+6061
+6062
+6063
+6086
+6110
+6111
+6112
+6159
+6343
+6346
+6347
+6350
+6379
+6432
+6444
+6445
+6513
+6514
+6515
+6566
+6600
+6601
+6602
+6619
+6622
+6653
+6665
+6666
+6667
+6668
+6669
+6679
+6697
+6888
+6969
+7023
+7262
+7272
+7400
+7401
+7402
+7473
+7474
+7478
+7542
+7547
+7624
+7631
+7687
+8008
+8074
+8080
+8118
+8123
+8140
+8194
+8243
+8280
+8880
+8883
+9001
+9006
+9080
+9100
+9101
+9102
+9103
+9119
+9306
+9312
+9389
+9418
+9535
+9536
+9695
+9800
+9899
+10000
+10010
+10050
+10051
+10110
+10212
+10933
+11001
+11112
+11371
+12222
+12223
+13075
+13720
+13721
+13724
+13782
+13783
+13785
+13786
+15345
+16482
+17500
+18104
+19283
+19315
+19812
+19813
+19814
+19999
+20000
+24465
+24554
+27000
+27001
+27002
+27003
+27004
+27005
+27006
+27007
+27008
+27009
+31457
+32400
+33434
+35357
+40000
+44818
+47001
+47808
+49151