MembersMgmtCommAttack.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. import os
  2. import sys
  3. from collections import deque
  4. from datetime import datetime
  5. from random import randint, randrange, choice, uniform
  6. import ID2TLib.Botnet.libbotnetcomm as lb
  7. from lea import Lea
  8. from scapy.layers.inet import IP, IPOption_Security
  9. import ID2TLib.Botnet.Message as Bmsg
  10. import ID2TLib.Utility as Util
  11. from Attack import BaseAttack
  12. from Attack.AttackParameters import Parameter as Param
  13. from Attack.AttackParameters import ParameterTypes
  14. from ID2TLib import Generator
  15. from ID2TLib.Botnet.CommunicationProcessor import CommunicationProcessor
  16. from ID2TLib.Botnet.MessageMapping import MessageMapping
  17. from ID2TLib.PcapAddressOperations import PcapAddressOperations
  18. from ID2TLib.Ports import PortSelectors
  19. class MembersMgmtCommAttack(BaseAttack.BaseAttack):
  20. def __init__(self):
  21. """
  22. Creates a new instance of the Membership Management Communication.
  23. """
  24. # Initialize communication
  25. super(MembersMgmtCommAttack, self).__init__(
  26. "Membership Management Communication Attack (MembersMgmtCommAttack)",
  27. "Injects Membership Management Communication", "Botnet communication")
  28. # Define allowed parameters and their type
  29. self.supported_params = {
  30. # parameters regarding attack
  31. Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
  32. Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
  33. Param.PACKETS_LIMIT: ParameterTypes.TYPE_INTEGER_POSITIVE,
  34. Param.ATTACK_DURATION: ParameterTypes.TYPE_INTEGER_POSITIVE,
  35. # use num_attackers to specify number of communicating devices?
  36. Param.NUMBER_INITIATOR_BOTS: ParameterTypes.TYPE_INTEGER_POSITIVE,
  37. # input file containing botnet communication
  38. Param.FILE_CSV: ParameterTypes.TYPE_FILEPATH,
  39. Param.FILE_XML: ParameterTypes.TYPE_FILEPATH,
  40. # the percentage of IP reuse (if total and other is specified, percentages are multiplied)
  41. Param.IP_REUSE_TOTAL: ParameterTypes.TYPE_PERCENTAGE,
  42. Param.IP_REUSE_LOCAL: ParameterTypes.TYPE_PERCENTAGE,
  43. Param.IP_REUSE_EXTERNAL: ParameterTypes.TYPE_PERCENTAGE,
  44. # the user-selected padding to add to every packet
  45. Param.PACKET_PADDING: ParameterTypes.TYPE_PADDING,
  46. # presence of NAT at the gateway of the network
  47. Param.NAT_PRESENT: ParameterTypes.TYPE_BOOLEAN,
  48. # whether the TTL distribution should be based on the input PCAP
  49. # or the CAIDA dataset
  50. Param.TTL_FROM_CAIDA: ParameterTypes.TYPE_BOOLEAN,
  51. # whether the destination port of a response should be the ephemeral port
  52. # its request came from or a static (server)port based on a hostname
  53. Param.MULTIPORT: ParameterTypes.TYPE_BOOLEAN,
  54. # information about the interval selection strategy
  55. Param.INTERVAL_SELECT_STRATEGY: ParameterTypes.TYPE_INTERVAL_SELECT_STRAT,
  56. Param.INTERVAL_SELECT_START: ParameterTypes.TYPE_INTEGER_POSITIVE,
  57. Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE,
  58. # determines whether injected packets are marked with an unused IP option
  59. # to easily filter them in e.g. wireshark
  60. Param.HIDDEN_MARK: ParameterTypes.TYPE_BOOLEAN
  61. }
  62. # create dict with MessageType values for fast name lookup
  63. self.msg_types = {}
  64. for msg_type in Bmsg.MessageType:
  65. self.msg_types[msg_type.value] = msg_type
  66. self.DEFAULT_XML_PATH = None
  67. def init_params(self):
  68. """
  69. Initialize some parameters of this communication-attack using the user supplied command line parameters.
  70. The remaining parameters are implicitly set in the provided data file. Note: the timestamps in the file
  71. have to be sorted in ascending order
  72. """
  73. # set class constants
  74. self.DEFAULT_XML_PATH = Util.RESOURCE_DIR + "Botnet/MembersMgmtComm_example.xml"
  75. # PARAMETERS: initialize with default values
  76. # (values are overwritten if user specifies them)
  77. self.add_param_value(Param.INJECT_AFTER_PACKET, 1 + randint(0, self.statistics.get_packet_count() // 5))
  78. self.add_param_value(Param.FILE_XML, self.DEFAULT_XML_PATH)
  79. # Alternatively new attack parameter?
  80. duration = int(float(self.statistics.get_capture_duration()))
  81. self.add_param_value(Param.ATTACK_DURATION, duration)
  82. self.add_param_value(Param.NUMBER_INITIATOR_BOTS, 1)
  83. # NAT on by default
  84. self.add_param_value(Param.NAT_PRESENT, True)
  85. # TODO: change 1 to something better
  86. self.add_param_value(Param.IP_REUSE_TOTAL, 1)
  87. self.add_param_value(Param.IP_REUSE_LOCAL, 0.5)
  88. self.add_param_value(Param.IP_REUSE_EXTERNAL, 0.5)
  89. # add default additional padding
  90. self.add_param_value(Param.PACKET_PADDING, 20)
  91. # choose the input PCAP as default base for the TTL distribution
  92. self.add_param_value(Param.TTL_FROM_CAIDA, False)
  93. # do not use multiple ports for requests and responses
  94. self.add_param_value(Param.MULTIPORT, False)
  95. # interval selection strategy
  96. self.add_param_value(Param.INTERVAL_SELECT_STRATEGY, "optimal")
  97. self.add_param_value(Param.HIDDEN_MARK, False)
  98. def generate_attack_pcap(self):
  99. """
  100. Injects the packets of this attack into a PCAP and stores it as a temporary file.
  101. :return: a tuple of the number packets injected, the path to the temporary attack PCAP
  102. and a list of additionally created files
  103. """
  104. # create the final messages that have to be sent, including all bot configurations
  105. messages = self._create_messages()
  106. if not messages:
  107. return 0, None
  108. # Setup (initial) parameters for packet creation loop
  109. buffer_size = 1000
  110. pkt_gen = Generator.PacketGenerator()
  111. padding = self.get_param_value(Param.PACKET_PADDING)
  112. packets = deque(maxlen=buffer_size)
  113. total_pkts = 0
  114. limit_packetcount = self.get_param_value(Param.PACKETS_LIMIT)
  115. limit_duration = self.get_param_value(Param.ATTACK_DURATION)
  116. path_attack_pcap = None
  117. over_thousand = False
  118. msg_packet_mapping = MessageMapping(messages, self.statistics.get_pcap_timestamp_start())
  119. mark_packets = self.get_param_value(Param.HIDDEN_MARK)
  120. # create packets to write to PCAP file
  121. for msg in messages:
  122. # retrieve the source and destination configurations
  123. ip_src, ip_dst = msg.src["IP"], msg.dst["IP"]
  124. mac_src, mac_dst = msg.src["MAC"], msg.dst["MAC"]
  125. if msg.type.is_request():
  126. port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
  127. else:
  128. port_src, port_dst = int(msg.src["DstPort"]), int(msg.dst["SrcPort"])
  129. ttl = int(msg.src["TTL"])
  130. # update duration
  131. duration = msg.time - messages[0].time
  132. # if total number of packets has been sent or the attack duration has been exceeded, stop
  133. if ((limit_packetcount is not None and total_pkts >= limit_packetcount) or
  134. (limit_duration is not None and duration >= limit_duration)):
  135. break
  136. # if the type of the message is a NL reply, determine the number of entries
  137. nl_size = 0
  138. if msg.type == Bmsg.MessageType.SALITY_NL_REPLY:
  139. nl_size = randint(1, 25) # what is max NL entries?
  140. # create suitable IP/UDP packet and add to packets list
  141. packet = pkt_gen.generate_mmcom_packet(ip_src=ip_src, ip_dst=ip_dst, ttl=ttl, mac_src=mac_src,
  142. mac_dst=mac_dst,
  143. port_src=port_src, port_dst=port_dst, message_type=msg.type,
  144. neighborlist_entries=nl_size)
  145. Generator.add_padding(packet, padding, True, True)
  146. packet.time = msg.time
  147. if mark_packets and isinstance(packet.payload, IP): # do this only for ip-packets
  148. ip_data = packet.payload
  149. hidden_opt = IPOption_Security()
  150. hidden_opt.option = 2 # "normal" security opt
  151. hidden_opt.security = 16 # magic value indicating NSA
  152. ip_data.options = hidden_opt
  153. packets.append(packet)
  154. msg_packet_mapping.map_message(msg, packet)
  155. total_pkts += 1
  156. # Store timestamp of first packet (for attack label)
  157. if total_pkts <= 1:
  158. self.attack_start_utime = packets[0].time
  159. elif total_pkts % buffer_size == 0: # every 1000 packets write them to the PCAP file (append)
  160. if over_thousand: # if over 1000 packets written, packet-length for the last few packets may differ
  161. packets = list(packets)
  162. Generator.equal_length(packets, length=max_len, padding=padding, force_len=True)
  163. last_packet = packets[-1]
  164. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  165. packets = deque(maxlen=buffer_size)
  166. else:
  167. packets = list(packets)
  168. Generator.equal_length(packets, padding=padding)
  169. last_packet = packets[-1]
  170. max_len = len(last_packet)
  171. over_thousand = True
  172. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  173. packets = deque(maxlen=buffer_size)
  174. # if there are unwritten packets remaining, write them to the PCAP file
  175. if len(packets) > 0:
  176. if over_thousand:
  177. packets = list(packets)
  178. Generator.equal_length(packets, length=max_len, padding=padding, force_len=True)
  179. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  180. last_packet = packets[-1]
  181. else:
  182. packets = list(packets)
  183. Generator.equal_length(packets, padding=padding)
  184. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  185. last_packet = packets[-1]
  186. # write the mapping to a file
  187. current_ts = datetime.now().strftime("%Y%m%d-%H%M%S")
  188. mapping_filename = "mapping_" + current_ts + ".xml"
  189. msg_packet_mapping.write_to_file(mapping_filename)
  190. # Store timestamp of last packet
  191. self.attack_end_utime = last_packet.time
  192. # Return packets sorted by packet by timestamp and total number of packets (sent)
  193. return total_pkts, path_attack_pcap, [mapping_filename]
  194. def generate_attack_packets(self):
  195. pass
  196. def _create_messages(self):
  197. """
  198. Creates the messages that are to be injected into the PCAP.
  199. :return: the final messages as a list
  200. """
  201. def add_ids_to_config(ids_to_add: list, existing_ips: list, new_ips: list, bot_configs: dict,
  202. idtype: str = "local", router_mac: str = ""):
  203. """
  204. Creates IP and MAC configurations for the given IDs and adds them to the existing configurations object.
  205. :param ids_to_add: all sorted IDs that have to be configured and added
  206. :param existing_ips: the existing IPs in the PCAP file that should be assigned to some, or all, IDs
  207. :param new_ips: the newly generated IPs that should be assigned to some, or all, IDs
  208. :param bot_configs: the existing configurations for the bots
  209. :param idtype: the locality type of the IDs
  210. :param router_mac: the MAC address of the router in the PCAP
  211. """
  212. ids = ids_to_add.copy()
  213. # macgen only needed, when IPs are new local IPs (therefore creating the object here suffices for the
  214. # current callers to not end up with the same MAC paired with different IPs)
  215. macgen = Generator.MacAddressGenerator()
  216. # assign existing IPs and the corresponding MAC addresses in the PCAP to the IDs
  217. for ip in existing_ips:
  218. random_id = choice(ids)
  219. mac = self.statistics.process_db_query("macAddress(IPAddress=%s)" % ip)
  220. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  221. ids.remove(random_id)
  222. # assign new IPs and for local IPs new MACs or for external IPs the router MAC to the IDs
  223. for ip in new_ips:
  224. random_id = choice(ids)
  225. if idtype == "local":
  226. mac = macgen.random_mac()
  227. elif idtype == "external":
  228. mac = router_mac
  229. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  230. ids.remove(random_id)
  231. def assign_realistic_ttls(bot_configs: dict):
  232. """
  233. Assigns a realisitic ttl to each bot from @param: bot_configs. Uses statistics and distribution to be able
  234. to calculate a realisitc ttl.
  235. :param bot_configs: List that contains all bots that should be assigned with realistic ttls.
  236. """
  237. ids = sorted(bot_configs.keys())
  238. for pos, bot in enumerate(ids):
  239. bot_type = bot_configs[bot]["Type"]
  240. if bot_type == "local": # Set fix TTL for local Bots
  241. bot_configs[bot]["TTL"] = 128
  242. # Set TTL based on TTL distribution of IP address
  243. else: # Set varying TTl for external Bots
  244. bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
  245. if len(bot_ttl_dist) > 0:
  246. source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
  247. bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
  248. else:
  249. most_used_ttl = self.statistics.process_db_query("most_used(ttlValue)")
  250. if isinstance(most_used_ttl, list):
  251. bot_configs[bot]["TTL"] = choice(self.statistics.process_db_query("most_used(ttlValue)"))
  252. else:
  253. bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
  254. def assign_realistic_timestamps(messages: list, external_ids: set, local_ids: set, avg_delay_local: list,
  255. avg_delay_external: list, zero_reference: float):
  256. """
  257. Assigns realistic timestamps to a set of messages
  258. :param messages: the set of messages to be updated
  259. :param external_ids: the set of bot ids, that are outside the network, i.e. external
  260. :param local_ids: the set of bot ids, that are inside the network, i.e. local
  261. :param avg_delay_local: the avg_delay distribution between the dispatch and the reception of a packet
  262. between local computers
  263. :param avg_delay_external: the avg_delay distribution between the dispatch and the reception of a packet
  264. between a local and an external computer
  265. :param zero_reference: the timestamp which is regarded as the beginning of the pcap_file and therefore
  266. handled like a timestamp that resembles 0
  267. """
  268. updated_msgs = []
  269. # Dict, takes a tuple of 2 Bot_IDs as a key (requester, responder), returns the time of the last response,
  270. # the requester received necessary in order to make sure, that additional requests are sent only after the
  271. # response to the last one was received
  272. last_response = {}
  273. for m in messages: # init
  274. last_response[(m.src, m.dst)] = -1
  275. # update all timestamps
  276. for req_msg in messages:
  277. if req_msg in updated_msgs:
  278. # message already updated
  279. continue
  280. # if req_msg.timestamp would be before the timestamp of the response to the last request, req_msg needs
  281. # to be sent later (else branch)
  282. if last_response[(req_msg.src, req_msg.dst)] == -1 or last_response[(req_msg.src, req_msg.dst)] < (
  283. zero_reference + req_msg.time - 0.05):
  284. # update req_msg timestamp with a variation of up to 50ms
  285. req_msg.time = zero_reference + req_msg.time + uniform(-0.05, 0.05)
  286. updated_msgs.append(req_msg)
  287. else:
  288. req_msg.time = last_response[(req_msg.src, req_msg.dst)] + 0.06 + uniform(-0.05, 0.05)
  289. # update response if necessary
  290. if req_msg.refer_msg_id != -1:
  291. respns_msg = messages[req_msg.refer_msg_id]
  292. # check for local or external communication and update response timestamp with the respective
  293. # avg delay
  294. if req_msg.src in external_ids or req_msg.dst in external_ids and avg_delay_external:
  295. # external communication
  296. external_dist = Lea.fromSeq(avg_delay_external)
  297. respns_msg.time = req_msg.time + float(external_dist.random()) * 0.001
  298. else:
  299. # local communication
  300. local_dist = Lea.fromSeq(avg_delay_local)
  301. respns_msg.time = req_msg.time + float(local_dist.random()) * 0.001
  302. updated_msgs.append(respns_msg)
  303. last_response[(req_msg.src, req_msg.dst)] = respns_msg.time
  304. def assign_ttls_from_caida(bot_configs):
  305. """
  306. Assign realistic TTL values to bots with respect to their IP, based on the CAIDA dataset.
  307. If there exists an entry for a bot's IP, the TTL is chosen based on a distribution over all used TTLs by
  308. this IP.
  309. If there is no such entry, the TTL is chosen based on a distribution over all used TTLs and their
  310. respective frequency.
  311. :param bot_configs: the existing bot configurations
  312. """
  313. def get_ip_ttl_distrib():
  314. """
  315. Parses the CSV file containing a mapping between IP and their used TTLs.
  316. :return: returns a dict with the IPs as keys and dicts for their TTL distribution as values
  317. """
  318. ip_based_distrib = {}
  319. with open("resources/CaidaTTL_perIP.csv", "r") as file:
  320. # every line consists of: IP, TTL, Frequency
  321. next(file) # skip CSV header line
  322. for line in file:
  323. ip_addr, ttl, freq = line.split(",")
  324. if ip_addr not in ip_based_distrib:
  325. # the values for ip_based_distrib are dicts with key=TTL, value=Frequency
  326. ip_based_distrib[ip_addr] = {}
  327. ip_based_distrib[ip_addr][ttl] = int(freq)
  328. return ip_based_distrib
  329. def get_total_ttl_distrib():
  330. """
  331. Parses the CSV file containing an overview of all used TTLs and their respective frequency.
  332. :return: returns a dict with the TTLs as keys and their frequencies as keys
  333. """
  334. total_ttl_distrib = {}
  335. with open("resources/CaidaTTL_total.csv", "r") as file:
  336. # every line consists of: TTL, Frequency, Fraction
  337. next(file) # skip CSV header line
  338. for line in file:
  339. ttl, freq, _ = line.split(",")
  340. total_ttl_distrib[ttl] = int(freq)
  341. return total_ttl_distrib
  342. # get the TTL distribution for every IP that is available in "resources/CaidaTTL_perIP.csv"
  343. ip_ttl_distrib = get_ip_ttl_distrib()
  344. # build a probability dict for the total TTL distribution
  345. total_ttl_prob_dict = Lea.fromValFreqsDict(get_total_ttl_distrib())
  346. # loop over every bot id and assign a TTL to the respective bot
  347. for bot_id in sorted(bot_configs):
  348. bot_type = bot_configs[bot_id]["Type"]
  349. bot_ip = bot_configs[bot_id]["IP"]
  350. if bot_type == "local":
  351. bot_configs[bot_id]["TTL"] = 128
  352. # if there exists detailed information about the TTL distribution of this IP
  353. elif bot_ip in ip_ttl_distrib:
  354. ip_ttl_freqs = ip_ttl_distrib[bot_ip]
  355. # build a probability dict from this IP's TTL distribution
  356. source_ttl_prob_dict = Lea.fromValFreqsDict(ip_ttl_freqs)
  357. bot_configs[bot_id]["TTL"] = source_ttl_prob_dict.random()
  358. # otherwise assign a random TTL based on the total TTL distribution
  359. else:
  360. bot_configs[bot_id]["TTL"] = total_ttl_prob_dict.random()
  361. # parse input CSV or XML
  362. filepath_xml = self.get_param_value(Param.FILE_XML)
  363. filepath_csv = self.get_param_value(Param.FILE_CSV)
  364. # use C++ communication processor for faster interval finding
  365. cpp_comm_proc = lb.botnet_comm_processor()
  366. # only use CSV input if the XML path is the default one
  367. # --> prefer XML input over CSV input (in case both are given)
  368. print_updates = False
  369. if filepath_csv and filepath_xml == self.DEFAULT_XML_PATH:
  370. filename = os.path.splitext(os.path.basename(filepath_csv))[0]
  371. filesize = os.path.getsize(filepath_csv) / 2**20 # get filesize in MB
  372. if filesize > 10:
  373. print("\nParsing input CSV file...", end=" ")
  374. sys.stdout.flush()
  375. print_updates = True
  376. cpp_comm_proc.parse_csv(filepath_csv)
  377. if print_updates:
  378. print("done.")
  379. print("Writing corresponding XML file...", end=" ")
  380. sys.stdout.flush()
  381. filepath_xml = cpp_comm_proc.write_xml(Util.OUT_DIR, filename)
  382. if print_updates:
  383. print("done.")
  384. else:
  385. filesize = os.path.getsize(filepath_xml) / 2**20 # get filesize in MB
  386. if filesize > 10:
  387. print("Parsing input XML file...", end=" ")
  388. sys.stdout.flush()
  389. print_updates = True
  390. cpp_comm_proc.parse_xml(filepath_xml)
  391. if print_updates:
  392. print("done.")
  393. # find a good communication mapping in the input file that matches the users parameters
  394. nat = self.get_param_value(Param.NAT_PRESENT)
  395. comm_proc = CommunicationProcessor(self.msg_types, nat)
  396. duration = self.get_param_value(Param.ATTACK_DURATION)
  397. number_init_bots = self.get_param_value(Param.NUMBER_INITIATOR_BOTS)
  398. strategy = self.get_param_value(Param.INTERVAL_SELECT_STRATEGY)
  399. start_idx = self.get_param_value(Param.INTERVAL_SELECT_START)
  400. end_idx = self.get_param_value(Param.INTERVAL_SELECT_END)
  401. potential_long_find_time = (
  402. strategy == "optimal" and (filesize > 4 and self.statistics.get_packet_count() > 1000))
  403. if print_updates or potential_long_find_time:
  404. if not print_updates:
  405. print()
  406. print("Selecting communication interval from input CSV/XML file...", end=" ")
  407. sys.stdout.flush()
  408. if potential_long_find_time:
  409. print("\nWarning: Because of the large input files and the (chosen) interval selection strategy")
  410. print("'optimal', this may take a while. Consider using selection strategy 'random' or 'custom'...",
  411. end=" ")
  412. sys.stdout.flush()
  413. print_updates = True
  414. comm_interval = comm_proc.get_comm_interval(cpp_comm_proc, strategy, number_init_bots, duration, start_idx,
  415. end_idx)
  416. if not comm_interval:
  417. print("Error: An interval that satisfies the input cannot be found.")
  418. return []
  419. if print_updates:
  420. print("done.") # print corresponding message to interval finding message
  421. # retrieve the mapping information
  422. mapped_ids = comm_interval["IDs"]
  423. packet_start_idx = comm_interval["Start"]
  424. packet_end_idx = comm_interval["End"]
  425. while len(mapped_ids) > number_init_bots:
  426. rm_idx = randrange(0, len(mapped_ids))
  427. del mapped_ids[rm_idx]
  428. if print_updates:
  429. print("Generating attack packets...", end=" ")
  430. sys.stdout.flush()
  431. # get the messages contained in the chosen interval
  432. abstract_packets = cpp_comm_proc.get_messages(packet_start_idx, packet_end_idx)
  433. comm_proc.set_mapping(abstract_packets, mapped_ids)
  434. # determine ID roles and select the messages that are to be mapped into the PCAP
  435. messages = comm_proc.det_id_roles_and_msgs()
  436. # use the previously detetermined roles to assign the locality of all IDs
  437. local_ids, external_ids = comm_proc.det_ext_and_local_ids()
  438. # determine number of reused local and external IPs
  439. reuse_percent_total = self.get_param_value(Param.IP_REUSE_TOTAL)
  440. reuse_percent_external = self.get_param_value(Param.IP_REUSE_EXTERNAL)
  441. reuse_percent_local = self.get_param_value(Param.IP_REUSE_LOCAL)
  442. # create IP and MAC configurations for the IDs/Bots
  443. ipgen = Generator.IPGenerator()
  444. pcapops = PcapAddressOperations(self.statistics)
  445. router_mac = pcapops.get_probable_router_mac()
  446. bot_configs = {}
  447. # retrieve and assign the IPs and MACs for the bots with respect to the given parameters
  448. # (IDs are always added to bot_configs in the same order under a given seed)
  449. number_local_ids, number_external_ids = len(local_ids), len(external_ids)
  450. # assign addresses for local IDs
  451. if number_local_ids > 0:
  452. reuse_count_local = int(reuse_percent_total * reuse_percent_local * number_local_ids)
  453. existing_local_ips = sorted(pcapops.get_existing_local_ips(reuse_count_local))
  454. new_local_ips = sorted(pcapops.get_new_local_ips(number_local_ids - len(existing_local_ips)))
  455. add_ids_to_config(sorted(local_ids), existing_local_ips, new_local_ips, bot_configs)
  456. # assign addresses for external IDs
  457. if number_external_ids > 0:
  458. reuse_count_external = int(reuse_percent_total * reuse_percent_external * number_external_ids)
  459. existing_external_ips = sorted(pcapops.get_existing_external_ips(reuse_count_external))
  460. remaining = len(external_ids) - len(existing_external_ips)
  461. for external_ip in existing_external_ips:
  462. ipgen.add_to_blacklist(external_ip)
  463. new_external_ips = sorted([ipgen.random_ip() for _ in range(remaining)])
  464. add_ids_to_config(sorted(external_ids), existing_external_ips, new_external_ips, bot_configs,
  465. idtype="external", router_mac=router_mac)
  466. # this is the timestamp at which the first packet should be injected, the packets have to be shifted to
  467. # the beginning of the pcap file (INJECT_AT_TIMESTAMP) and then the offset of the packets have to be
  468. # compensated to start at the given point in time
  469. zero_reference = self.get_param_value(Param.INJECT_AT_TIMESTAMP) - messages[0].time
  470. # calculate the average delay values for local and external responses
  471. avg_delay_local, avg_delay_external = self.statistics.get_avg_delay_distributions(False)
  472. # set timestamps
  473. assign_realistic_timestamps(messages, external_ids, local_ids, avg_delay_local, avg_delay_external,
  474. zero_reference)
  475. port_selector = PortSelectors.LINUX
  476. reserved_ports = set(int(line.strip()) for line in open(Util.RESOURCE_DIR + "reserved_ports.txt").readlines())
  477. def filter_reserved(get_port):
  478. port = get_port()
  479. while port in reserved_ports:
  480. port = get_port()
  481. return port
  482. # create port configurations for the bots
  483. use_multiple_ports = self.get_param_value(Param.MULTIPORT)
  484. for bot in sorted(bot_configs):
  485. bot_configs[bot]["SrcPort"] = filter_reserved(port_selector.select_port_udp)
  486. if not use_multiple_ports:
  487. bot_configs[bot]["DstPort"] = filter_reserved(Generator.gen_random_server_port)
  488. else:
  489. bot_configs[bot]["DstPort"] = filter_reserved(port_selector.select_port_udp)
  490. # assign realistic TTL for every bot
  491. if self.get_param_value(Param.TTL_FROM_CAIDA):
  492. assign_ttls_from_caida(bot_configs)
  493. else:
  494. assign_realistic_ttls(bot_configs)
  495. # put together the final messages including the full sender and receiver
  496. # configurations (i.e. IP, MAC, port, ...) for easier later use
  497. final_messages = []
  498. messages = sorted(messages, key=lambda m: m.time)
  499. new_id = 0
  500. for msg in messages:
  501. type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]
  502. id_src, id_dst = msg.src, msg.dst
  503. # sort out messages that do not have a suitable locality setting
  504. if type_src == "external" and type_dst == "external":
  505. continue
  506. msg.src, msg.dst = bot_configs[id_src], bot_configs[id_dst]
  507. msg.src["ID"], msg.dst["ID"] = id_src, id_dst
  508. msg.msg_id = new_id
  509. new_id += 1
  510. # Important here to update refers, if needed later?
  511. final_messages.append(msg)
  512. return final_messages