3
0

MembersMgmtCommAttack.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. from enum import Enum
  2. from random import randint, randrange, choice, uniform
  3. from collections import deque
  4. from scipy.stats import gamma
  5. from lea import Lea
  6. from datetime import datetime
  7. import os
  8. import sys
  9. import ID2TLib.Botnet.libbotnetcomm as lb
  10. from Attack import BaseAttack
  11. from Attack.AttackParameters import Parameter as Param
  12. from Attack.AttackParameters import ParameterTypes
  13. from ID2TLib.Ports import PortSelectors
  14. import ID2TLib.Utility as Util
  15. class MessageType(Enum):
  16. """
  17. Defines possible botnet message types
  18. """
  19. TIMEOUT = 3
  20. SALITY_NL_REQUEST = 101
  21. SALITY_NL_REPLY = 102
  22. SALITY_HELLO = 103
  23. SALITY_HELLO_REPLY = 104
  24. def is_request(mtype):
  25. """
  26. Checks whether the given message type is a request or not.
  27. :param mtype: the message type to check
  28. :return: True if it is a request, False otherwise
  29. """
  30. return mtype in {MessageType.SALITY_HELLO, MessageType.SALITY_NL_REQUEST}
  31. def is_response(mtype):
  32. """
  33. Checks whether the given message type is a response or not.
  34. :param mtype: the message type to check
  35. :return: True if it is a response, False otherwise
  36. """
  37. return mtype in {MessageType.SALITY_HELLO_REPLY, MessageType.SALITY_NL_REPLY}
  38. class Message():
  39. INVALID_LINENO = -1
  40. """
  41. Defines a compact message type that contains all necessary information.
  42. """
  43. def __init__(self, msg_id: int, src, dst, type_: MessageType, time: float, refer_msg_id: int=-1, line_no = -1):
  44. """
  45. Constructs a message with the given parameters.
  46. :param msg_id: the ID of the message
  47. :param src: something identifiying the source, e.g. ID or configuration
  48. :param dst: something identifiying the destination, e.g. ID or configuration
  49. :param type_: the type of the message
  50. :param time: the timestamp of the message
  51. :param refer_msg_id: the ID this message is a request for or reply to. -1 if there is no related message.
  52. :param line_no: The line number this message appeared at in the original CSV file
  53. """
  54. self.msg_id = msg_id
  55. self.src = src
  56. self.dst = dst
  57. self.type = type_
  58. self.time = time
  59. self.csv_time = time
  60. self.refer_msg_id = refer_msg_id
  61. self.line_no = line_no
  62. def __str__(self):
  63. str_ = "{0}. at {1}: {2}-->{3}, {4}, refer:{5} (line {6})".format(self.msg_id, self.time, self.src, self.dst, self.type, self.refer_msg_id, self.line_no)
  64. return str_
  65. from ID2TLib import FileUtils, Generator
  66. from ID2TLib.IPv4 import IPAddress
  67. from ID2TLib.PcapAddressOperations import PcapAddressOperations
  68. from ID2TLib.Botnet.CommunicationProcessor import CommunicationProcessor
  69. from ID2TLib.Botnet.MessageMapping import MessageMapping
  70. from ID2TLib.PcapFile import PcapFile
  71. from Core.Statistics import Statistics
  72. from scapy.layers.inet import IP, IPOption_Security
  73. class MembersMgmtCommAttack(BaseAttack.BaseAttack):
  74. def __init__(self):
  75. """
  76. Creates a new instance of the Membership Management Communication.
  77. """
  78. # Initialize communication
  79. super(MembersMgmtCommAttack, self).__init__("Membership Management Communication Attack (MembersMgmtCommAttack)",
  80. "Injects Membership Management Communication", "Botnet communication")
  81. # Define allowed parameters and their type
  82. self.supported_params = {
  83. # parameters regarding attack
  84. Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
  85. Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
  86. Param.PACKETS_LIMIT: ParameterTypes.TYPE_INTEGER_POSITIVE,
  87. Param.ATTACK_DURATION: ParameterTypes.TYPE_INTEGER_POSITIVE,
  88. # use num_attackers to specify number of communicating devices?
  89. Param.NUMBER_INITIATOR_BOTS: ParameterTypes.TYPE_INTEGER_POSITIVE,
  90. # input file containing botnet communication
  91. Param.FILE_CSV: ParameterTypes.TYPE_FILEPATH,
  92. Param.FILE_XML: ParameterTypes.TYPE_FILEPATH,
  93. # the percentage of IP reuse (if total and other is specified, percentages are multiplied)
  94. Param.IP_REUSE_TOTAL: ParameterTypes.TYPE_PERCENTAGE,
  95. Param.IP_REUSE_LOCAL: ParameterTypes.TYPE_PERCENTAGE,
  96. Param.IP_REUSE_EXTERNAL: ParameterTypes.TYPE_PERCENTAGE,
  97. # the user-selected padding to add to every packet
  98. Param.PACKET_PADDING: ParameterTypes.TYPE_PADDING,
  99. # presence of NAT at the gateway of the network
  100. Param.NAT_PRESENT: ParameterTypes.TYPE_BOOLEAN,
  101. # whether the TTL distribution should be based on the input PCAP
  102. # or the CAIDA dataset
  103. Param.TTL_FROM_CAIDA: ParameterTypes.TYPE_BOOLEAN,
  104. # whether the destination port of a response should be the ephemeral port
  105. # its request came from or a static (server)port based on a hostname
  106. Param.MULTIPORT: ParameterTypes.TYPE_BOOLEAN,
  107. # information about the interval selection strategy
  108. Param.INTERVAL_SELECT_STRATEGY: ParameterTypes.TYPE_INTERVAL_SELECT_STRAT,
  109. Param.INTERVAL_SELECT_START: ParameterTypes.TYPE_INTEGER_POSITIVE,
  110. Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE,
  111. # determines whether injected packets are marked with an unused IP option
  112. # to easily filter them in e.g. wireshark
  113. Param.HIDDEN_MARK: ParameterTypes.TYPE_BOOLEAN
  114. }
  115. # create dict with MessageType values for fast name lookup
  116. self.msg_types = {}
  117. for msg_type in MessageType:
  118. self.msg_types[msg_type.value] = msg_type
  119. def init_params(self):
  120. """
  121. Initialize some parameters of this communication-attack using the user supplied command line parameters.
  122. The remaining parameters are implicitly set in the provided data file. Note: the timestamps in the file
  123. have to be sorted in ascending order
  124. :param statistics: Reference to a statistics object.
  125. """
  126. # set class constants
  127. self.DEFAULT_XML_PATH = "resources/Botnet/MembersMgmtComm_example.xml"
  128. # PARAMETERS: initialize with default values
  129. # (values are overwritten if user specifies them)
  130. self.add_param_value(Param.INJECT_AFTER_PACKET, 1 + randint(0, self.statistics.get_packet_count() // 5))
  131. self.add_param_value(Param.FILE_XML, self.DEFAULT_XML_PATH)
  132. # Alternatively new attack parameter?
  133. duration = int(float(self.statistics.get_capture_duration()))
  134. self.add_param_value(Param.ATTACK_DURATION, duration)
  135. self.add_param_value(Param.NUMBER_INITIATOR_BOTS, 1)
  136. # NAT on by default
  137. self.add_param_value(Param.NAT_PRESENT, True)
  138. # TODO: change 1 to something better
  139. self.add_param_value(Param.IP_REUSE_TOTAL, 1)
  140. self.add_param_value(Param.IP_REUSE_LOCAL, 0.5)
  141. self.add_param_value(Param.IP_REUSE_EXTERNAL, 0.5)
  142. # add default additional padding
  143. self.add_param_value(Param.PACKET_PADDING, 20)
  144. # choose the input PCAP as default base for the TTL distribution
  145. self.add_param_value(Param.TTL_FROM_CAIDA, False)
  146. # do not use multiple ports for requests and responses
  147. self.add_param_value(Param.MULTIPORT, False)
  148. # interval selection strategy
  149. self.add_param_value(Param.INTERVAL_SELECT_STRATEGY, "optimal")
  150. self.add_param_value(Param.HIDDEN_MARK, False)
  151. def generate_attack_pcap(self):
  152. """
  153. Injects the packets of this attack into a PCAP and stores it as a temporary file.
  154. :return: a tuple of the number packets injected and the path to the temporary attack PCAP
  155. """
  156. # create the final messages that have to be sent, including all bot configurations
  157. messages = self._create_messages()
  158. if messages == []:
  159. return 0, None
  160. # Setup (initial) parameters for packet creation loop
  161. BUFFER_SIZE = 1000
  162. pkt_gen = Generator.PacketGenerator()
  163. padding = self.get_param_value(Param.PACKET_PADDING)
  164. packets = deque(maxlen=BUFFER_SIZE)
  165. total_pkts = 0
  166. limit_packetcount = self.get_param_value(Param.PACKETS_LIMIT)
  167. limit_duration = self.get_param_value(Param.ATTACK_DURATION)
  168. path_attack_pcap = None
  169. overThousand = False
  170. msg_packet_mapping = MessageMapping(messages, self.statistics.get_pcap_timestamp_start())
  171. mark_packets = self.get_param_value(Param.HIDDEN_MARK)
  172. # create packets to write to PCAP file
  173. for msg in messages:
  174. # retrieve the source and destination configurations
  175. id_src, id_dst = msg.src["ID"], msg.dst["ID"]
  176. ip_src, ip_dst = msg.src["IP"], msg.dst["IP"]
  177. mac_src, mac_dst = msg.src["MAC"], msg.dst["MAC"]
  178. if msg.type.is_request():
  179. port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
  180. else:
  181. port_src, port_dst = int(msg.src["DstPort"]), int(msg.dst["SrcPort"])
  182. ttl = int(msg.src["TTL"])
  183. # update duration
  184. duration = msg.time - messages[0].time
  185. # if total number of packets has been sent or the attack duration has been exceeded, stop
  186. if ((limit_packetcount is not None and total_pkts >= limit_packetcount) or
  187. (limit_duration is not None and duration >= limit_duration)):
  188. break
  189. # if the type of the message is a NL reply, determine the number of entries
  190. nl_size = 0
  191. if msg.type == MessageType.SALITY_NL_REPLY:
  192. nl_size = randint(1, 25) # what is max NL entries?
  193. # create suitable IP/UDP packet and add to packets list
  194. packet = pkt_gen.generate_mmcom_packet(ip_src=ip_src, ip_dst=ip_dst, ttl=ttl, mac_src=mac_src, mac_dst=mac_dst,
  195. port_src=port_src, port_dst=port_dst, message_type=msg.type, neighborlist_entries=nl_size)
  196. Generator.add_padding(packet, padding,True, True)
  197. packet.time = msg.time
  198. if mark_packets and isinstance(packet.payload, IP): # do this only for ip-packets
  199. ip_data = packet.payload
  200. hidden_opt = IPOption_Security()
  201. hidden_opt.option = 2 # "normal" security opt
  202. hidden_opt.security = 16 # magic value indicating NSA
  203. ip_data.options = hidden_opt
  204. packets.append(packet)
  205. msg_packet_mapping.map_message(msg, packet)
  206. total_pkts += 1
  207. # Store timestamp of first packet (for attack label)
  208. if total_pkts <= 1:
  209. self.attack_start_utime = packets[0].time
  210. elif total_pkts % BUFFER_SIZE == 0: # every 1000 packets write them to the PCAP file (append)
  211. if overThousand: # if over 1000 packets written, there may be a different packet-length for the last few packets
  212. packets = list(packets)
  213. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  214. last_packet = packets[-1]
  215. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  216. packets = deque(maxlen=BUFFER_SIZE)
  217. else:
  218. packets = list(packets)
  219. Generator.equal_length(packets, padding = padding)
  220. last_packet = packets[-1]
  221. max_len = len(last_packet)
  222. overThousand = True
  223. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  224. packets = deque(maxlen=BUFFER_SIZE)
  225. # if there are unwritten packets remaining, write them to the PCAP file
  226. if len(packets) > 0:
  227. if overThousand:
  228. packets = list(packets)
  229. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  230. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  231. last_packet = packets[-1]
  232. else:
  233. packets = list(packets)
  234. Generator.equal_length(packets, padding = padding)
  235. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  236. last_packet = packets[-1]
  237. # write the mapping to a file
  238. current_ts = datetime.now().strftime("%Y%m%d-%H%M%S")
  239. mapping_filename = "mapping_" + current_ts
  240. msg_packet_mapping.write_to_file(mapping_filename)
  241. Util.MISC_OUT_FILES["mapping.xml"] = mapping_filename
  242. # Store timestamp of last packet
  243. self.attack_end_utime = last_packet.time
  244. # Return packets sorted by packet by timestamp and total number of packets (sent)
  245. return total_pkts , path_attack_pcap
  246. def generate_attack_packets(self):
  247. pass
  248. def _create_messages(self):
  249. """
  250. Creates the messages that are to be injected into the PCAP.
  251. :return: the final messages as a list
  252. """
  253. def add_ids_to_config(ids_to_add: list, existing_ips: list, new_ips: list, bot_configs: dict, idtype:str="local", router_mac:str=""):
  254. """
  255. Creates IP and MAC configurations for the given IDs and adds them to the existing configurations object.
  256. :param ids_to_add: all sorted IDs that have to be configured and added
  257. :param existing_ips: the existing IPs in the PCAP file that should be assigned to some, or all, IDs
  258. :param new_ips: the newly generated IPs that should be assigned to some, or all, IDs
  259. :param bot_configs: the existing configurations for the bots
  260. :param idtype: the locality type of the IDs
  261. :param router_mac: the MAC address of the router in the PCAP
  262. """
  263. ids = ids_to_add.copy()
  264. # macgen only needed, when IPs are new local IPs (therefore creating the object here suffices for the current callers
  265. # to not end up with the same MAC paired with different IPs)
  266. macgen = Generator.MacAddressGenerator()
  267. # assign existing IPs and the corresponding MAC addresses in the PCAP to the IDs
  268. for ip in existing_ips:
  269. random_id = choice(ids)
  270. mac = self.statistics.process_db_query("macAddress(IPAddress=%s)" % ip)
  271. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  272. ids.remove(random_id)
  273. # assign new IPs and for local IPs new MACs or for external IPs the router MAC to the IDs
  274. for ip in new_ips:
  275. random_id = choice(ids)
  276. if idtype == "local":
  277. mac = macgen.random_mac()
  278. elif idtype == "external":
  279. mac = router_mac
  280. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  281. ids.remove(random_id)
  282. def assign_realistic_ttls(bot_configs:list):
  283. '''
  284. Assigns a realisitic ttl to each bot from @param: bot_configs. Uses statistics and distribution to be able
  285. to calculate a realisitc ttl.
  286. :param bot_configs: List that contains all bots that should be assigned with realistic ttls.
  287. '''
  288. ids = sorted(bot_configs.keys())
  289. for pos,bot in enumerate(ids):
  290. bot_type = bot_configs[bot]["Type"]
  291. if(bot_type == "local"): # Set fix TTL for local Bots
  292. bot_configs[bot]["TTL"] = 128
  293. # Set TTL based on TTL distribution of IP address
  294. else: # Set varying TTl for external Bots
  295. bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
  296. if len(bot_ttl_dist) > 0:
  297. source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
  298. bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
  299. else:
  300. most_used_ttl = self.statistics.process_db_query("most_used(ttlValue)")
  301. if isinstance(most_used_ttl, list):
  302. bot_configs[bot]["TTL"] = choice(self.statistics.process_db_query("most_used(ttlValue)"))
  303. else:
  304. bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
  305. def assign_realistic_timestamps(messages: list, external_ids: set, local_ids: set, avg_delay_local:float, avg_delay_external: float, zero_reference:float):
  306. """
  307. Assigns realistic timestamps to a set of messages
  308. :param messages: the set of messages to be updated
  309. :param external_ids: the set of bot ids, that are outside the network, i.e. external
  310. :param local_ids: the set of bot ids, that are inside the network, i.e. local
  311. :avg_delay_local: the avg_delay between the dispatch and the reception of a packet between local computers
  312. :avg_delay_external: the avg_delay between the dispatch and the reception of a packet between a local and an external computer
  313. :zero_reference: the timestamp which is regarded as the beginning of the pcap_file and therefore handled like a timestamp that resembles 0
  314. """
  315. updated_msgs = []
  316. last_response = {} # Dict, takes a tuple of 2 Bot_IDs as a key (requester, responder), returns the time of the last response, the requester received
  317. # necessary in order to make sure, that additional requests are sent only after the response to the last one was received
  318. for msg in messages: # init
  319. last_response[(msg.src, msg.dst)] = -1
  320. # update all timestamps
  321. for req_msg in messages:
  322. if(req_msg in updated_msgs):
  323. # message already updated
  324. continue
  325. # if req_msg.timestamp would be before the timestamp of the response to the last request, req_msg needs to be sent later (else branch)
  326. if last_response[(req_msg.src, req_msg.dst)] == -1 or last_response[(req_msg.src, req_msg.dst)] < (zero_reference + req_msg.time - 0.05):
  327. ## update req_msg timestamp with a variation of up to 50ms
  328. req_msg.time = zero_reference + req_msg.time + uniform(-0.05, 0.05)
  329. updated_msgs.append(req_msg)
  330. else:
  331. req_msg.time = last_response[(req_msg.src, req_msg.dst)] + 0.06 + uniform(-0.05, 0.05)
  332. # update response if necessary
  333. if req_msg.refer_msg_id != -1:
  334. respns_msg = messages[req_msg.refer_msg_id]
  335. # check for local or external communication and update response timestamp with the respective avg delay
  336. if req_msg.src in external_ids or req_msg.dst in external_ids:
  337. #external communication
  338. respns_msg.time = req_msg.time + avg_delay_external + uniform(-0.1*avg_delay_external, 0.1*avg_delay_external)
  339. else:
  340. #local communication
  341. respns_msg.time = req_msg.time + avg_delay_local + uniform(-0.1*avg_delay_local, 0.1*avg_delay_local)
  342. updated_msgs.append(respns_msg)
  343. last_response[(req_msg.src, req_msg.dst)] = respns_msg.time
  344. def assign_ttls_from_caida(bot_configs):
  345. """
  346. Assign realistic TTL values to bots with respect to their IP, based on the CAIDA dataset.
  347. If there exists an entry for a bot's IP, the TTL is chosen based on a distribution over all used TTLs by this IP.
  348. If there is no such entry, the TTL is chosen based on a distribution over all used TTLs and their respective frequency.
  349. :param bot_configs: the existing bot configurations
  350. """
  351. def get_ip_ttl_distrib():
  352. """
  353. Parses the CSV file containing a mapping between IP and their used TTLs.
  354. :return: returns a dict with the IPs as keys and dicts for their TTL disribution as values
  355. """
  356. ip_based_distrib = {}
  357. with open("resources/CaidaTTL_perIP.csv", "r") as file:
  358. # every line consists of: IP, TTL, Frequency
  359. next(file) # skip CSV header line
  360. for line in file:
  361. ip_addr, ttl, freq = line.split(",")
  362. if ip_addr not in ip_based_distrib:
  363. ip_based_distrib[ip_addr] = {} # the values for ip_based_distrib are dicts with key=TTL, value=Frequency
  364. ip_based_distrib[ip_addr][ttl] = int(freq)
  365. return ip_based_distrib
  366. def get_total_ttl_distrib():
  367. """
  368. Parses the CSV file containing an overview of all used TTLs and their respective frequency.
  369. :return: returns a dict with the TTLs as keys and their frequencies as keys
  370. """
  371. total_ttl_distrib = {}
  372. with open("resources/CaidaTTL_total.csv", "r") as file:
  373. # every line consists of: TTL, Frequency, Fraction
  374. next(file) # skip CSV header line
  375. for line in file:
  376. ttl, freq, _ = line.split(",")
  377. total_ttl_distrib[ttl] = int(freq)
  378. return total_ttl_distrib
  379. # get the TTL distribution for every IP that is available in "resources/CaidaTTL_perIP.csv"
  380. ip_ttl_distrib = get_ip_ttl_distrib()
  381. # build a probability dict for the total TTL distribution
  382. total_ttl_prob_dict = Lea.fromValFreqsDict(get_total_ttl_distrib())
  383. # loop over every bot id and assign a TTL to the respective bot
  384. for bot_id in sorted(bot_configs):
  385. bot_type = bot_configs[bot_id]["Type"]
  386. bot_ip = bot_configs[bot_id]["IP"]
  387. if bot_type == "local":
  388. bot_configs[bot_id]["TTL"] = 128
  389. # if there exists detailed information about the TTL distribution of this IP
  390. elif bot_ip in ip_ttl_distrib:
  391. ip_ttl_freqs = ip_ttl_distrib[bot_ip]
  392. source_ttl_prob_dict = Lea.fromValFreqsDict(ip_ttl_freqs) # build a probability dict from this IP's TTL distribution
  393. bot_configs[bot_id]["TTL"] = source_ttl_prob_dict.random()
  394. # otherwise assign a random TTL based on the total TTL distribution
  395. else:
  396. bot_configs[bot_id]["TTL"] = total_ttl_prob_dict.random()
  397. # parse input CSV or XML
  398. filepath_xml = self.get_param_value(Param.FILE_XML)
  399. filepath_csv = self.get_param_value(Param.FILE_CSV)
  400. # use C++ communication processor for faster interval finding
  401. cpp_comm_proc = lb.botnet_comm_processor();
  402. # only use CSV input if the XML path is the default one
  403. # --> prefer XML input over CSV input (in case both are given)
  404. print_updates = False
  405. if filepath_csv and filepath_xml == self.DEFAULT_XML_PATH:
  406. filename = os.path.splitext(os.path.basename(filepath_csv))[0]
  407. filesize = os.path.getsize(filepath_csv) / 2**20 # get filesize in MB
  408. if filesize > 10:
  409. print("\nParsing input CSV file...", end=" ")
  410. sys.stdout.flush()
  411. print_updates = True
  412. cpp_comm_proc.parse_csv(filepath_csv)
  413. if print_updates:
  414. print("done.")
  415. print("Writing corresponding XML file...", end=" ")
  416. sys.stdout.flush()
  417. filepath_xml = cpp_comm_proc.write_xml(Util.OUT_DIR, filename)
  418. Util.MISC_OUT_FILES[filepath_xml] = None
  419. if print_updates: print("done.")
  420. else:
  421. filesize = os.path.getsize(filepath_xml) / 2**20 # get filesize in MB
  422. if filesize > 10:
  423. print("Parsing input XML file...", end=" ")
  424. sys.stdout.flush()
  425. print_updates = True
  426. cpp_comm_proc.parse_xml(filepath_xml)
  427. if print_updates: print("done.")
  428. # find a good communication mapping in the input file that matches the users parameters
  429. nat = self.get_param_value(Param.NAT_PRESENT)
  430. comm_proc = CommunicationProcessor(self.msg_types, nat)
  431. duration = self.get_param_value(Param.ATTACK_DURATION)
  432. number_init_bots = self.get_param_value(Param.NUMBER_INITIATOR_BOTS)
  433. strategy = self.get_param_value(Param.INTERVAL_SELECT_STRATEGY)
  434. start_idx = self.get_param_value(Param.INTERVAL_SELECT_START)
  435. end_idx = self.get_param_value(Param.INTERVAL_SELECT_END)
  436. potential_long_find_time = (strategy == "optimal" and (filesize > 4 and self.statistics.get_packet_count() > 1000))
  437. if print_updates or potential_long_find_time:
  438. if not print_updates: print()
  439. print("Selecting communication interval from input CSV/XML file...", end=" ")
  440. sys.stdout.flush()
  441. if potential_long_find_time:
  442. print("\nWarning: Because of the large input files and the (chosen) interval selection strategy 'optimal',")
  443. print("this may take a while. Consider using selection strategy 'random' or 'custom'...", end=" ")
  444. sys.stdout.flush()
  445. print_updates = True
  446. comm_interval = comm_proc.get_comm_interval(cpp_comm_proc, strategy, number_init_bots, duration, start_idx, end_idx)
  447. if not comm_interval:
  448. print("Error: An interval that satisfies the input cannot be found.")
  449. return []
  450. if print_updates: print("done.") # print corresponding message to interval finding message
  451. # retrieve the mapping information
  452. mapped_ids, packet_start_idx, packet_end_idx = comm_interval["IDs"], comm_interval["Start"], comm_interval["End"]
  453. while len(mapped_ids) > number_init_bots:
  454. rm_idx = randrange(0, len(mapped_ids))
  455. del mapped_ids[rm_idx]
  456. if print_updates: print("Generating attack packets...", end=" ")
  457. sys.stdout.flush()
  458. # get the messages contained in the chosen interval
  459. abstract_packets = cpp_comm_proc.get_messages(packet_start_idx, packet_end_idx);
  460. comm_proc.set_mapping(abstract_packets, mapped_ids)
  461. # determine ID roles and select the messages that are to be mapped into the PCAP
  462. messages = comm_proc.det_id_roles_and_msgs()
  463. # use the previously detetermined roles to assign the locality of all IDs
  464. local_ids, external_ids = comm_proc.det_ext_and_local_ids()
  465. # determine number of reused local and external IPs
  466. reuse_percent_total = self.get_param_value(Param.IP_REUSE_TOTAL)
  467. reuse_percent_external = self.get_param_value(Param.IP_REUSE_EXTERNAL)
  468. reuse_percent_local = self.get_param_value(Param.IP_REUSE_LOCAL)
  469. reuse_count_external = int(reuse_percent_total * reuse_percent_external * len(mapped_ids))
  470. reuse_count_local = int(reuse_percent_total * reuse_percent_local * len(mapped_ids))
  471. # create IP and MAC configurations for the IDs/Bots
  472. ipgen = Generator.IPGenerator()
  473. pcapops = PcapAddressOperations(self.statistics)
  474. router_mac = pcapops.get_probable_router_mac()
  475. bot_configs = {}
  476. # retrieve and assign the IPs and MACs for the bots with respect to the given parameters
  477. # (IDs are always added to bot_configs in the same order under a given seed)
  478. number_local_ids, number_external_ids = len(local_ids), len(external_ids)
  479. # assign addresses for local IDs
  480. if number_local_ids > 0:
  481. reuse_count_local = int(reuse_percent_total * reuse_percent_local * number_local_ids)
  482. existing_local_ips = sorted(pcapops.get_existing_local_ips(reuse_count_local))
  483. new_local_ips = sorted(pcapops.get_new_local_ips(number_local_ids - len(existing_local_ips)))
  484. add_ids_to_config(sorted(local_ids), existing_local_ips, new_local_ips, bot_configs)
  485. # assign addresses for external IDs
  486. if number_external_ids > 0:
  487. reuse_count_external = int(reuse_percent_total * reuse_percent_external * number_external_ids)
  488. existing_external_ips = sorted(pcapops.get_existing_external_ips(reuse_count_external))
  489. remaining = len(external_ids) - len(existing_external_ips)
  490. for external_ip in existing_external_ips: ipgen.add_to_blacklist(external_ip)
  491. new_external_ips = sorted([ipgen.random_ip() for _ in range(remaining)])
  492. add_ids_to_config(sorted(external_ids), existing_external_ips, new_external_ips, bot_configs, idtype="external", router_mac=router_mac)
  493. # this is the timestamp at which the first packet should be injected, the packets have to be shifted to the beginning of the
  494. # pcap file (INJECT_AT_TIMESTAMP) and then the offset of the packets have to be compensated to start at the given point in time
  495. zero_reference = self.get_param_value(Param.INJECT_AT_TIMESTAMP) - messages[0].time
  496. # calculate the average delay values for local and external responses
  497. avg_delay_local, avg_delay_external = self.statistics.get_avg_delay_local_ext()
  498. #set timestamps
  499. assign_realistic_timestamps(messages, external_ids, local_ids, avg_delay_local, avg_delay_external, zero_reference)
  500. portSelector = PortSelectors.LINUX
  501. reserved_ports = set(int(line.strip()) for line in open("resources/reserved_ports.txt").readlines())
  502. def filter_reserved(get_port):
  503. port = get_port()
  504. while port in reserved_ports:
  505. port = get_port()
  506. return port
  507. # create port configurations for the bots
  508. use_multiple_ports = self.get_param_value(Param.MULTIPORT)
  509. for bot in sorted(bot_configs):
  510. bot_configs[bot]["SrcPort"] = filter_reserved(portSelector.select_port_udp)
  511. if not use_multiple_ports:
  512. bot_configs[bot]["DstPort"] = filter_reserved(Generator.gen_random_server_port)
  513. else:
  514. bot_configs[bot]["DstPort"] = filter_reserved(portSelector.select_port_udp)
  515. # assign realistic TTL for every bot
  516. if self.get_param_value(Param.TTL_FROM_CAIDA):
  517. assign_ttls_from_caida(bot_configs)
  518. else:
  519. assign_realistic_ttls(bot_configs)
  520. # put together the final messages including the full sender and receiver
  521. # configurations (i.e. IP, MAC, port, ...) for easier later use
  522. final_messages = []
  523. messages = sorted(messages, key=lambda msg: msg.time)
  524. new_id = 0
  525. for msg in messages:
  526. type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]
  527. id_src, id_dst = msg.src, msg.dst
  528. # sort out messages that do not have a suitable locality setting
  529. if type_src == "external" and type_dst == "external":
  530. continue
  531. msg.src, msg.dst = bot_configs[id_src], bot_configs[id_dst]
  532. msg.src["ID"], msg.dst["ID"] = id_src, id_dst
  533. msg.msg_id = new_id
  534. new_id += 1
  535. ### Important here to update refers, if needed later?
  536. final_messages.append(msg)
  537. return final_messages