MembersMgmtCommAttack.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665
  1. from enum import Enum
  2. from random import randint, randrange, choice, uniform
  3. from collections import deque
  4. from scipy.stats import gamma
  5. from lea import Lea
  6. from datetime import datetime
  7. import os
  8. import sys
  9. import ID2TLib.Botnet.libbotnetcomm as lb
  10. from Attack import BaseAttack
  11. from Attack.AttackParameters import Parameter as Param
  12. from Attack.AttackParameters import ParameterTypes
  13. from ID2TLib.Ports import PortSelectors
  14. import ID2TLib.Utility as Util
  15. class MessageType(Enum):
  16. """
  17. Defines possible botnet message types
  18. """
  19. TIMEOUT = 3
  20. SALITY_NL_REQUEST = 101
  21. SALITY_NL_REPLY = 102
  22. SALITY_HELLO = 103
  23. SALITY_HELLO_REPLY = 104
  24. def is_request(mtype):
  25. return mtype in {MessageType.SALITY_HELLO, MessageType.SALITY_NL_REQUEST}
  26. def is_response(mtype):
  27. return mtype in {MessageType.SALITY_HELLO_REPLY, MessageType.SALITY_NL_REPLY}
  28. class Message():
  29. INVALID_LINENO = -1
  30. """
  31. Defines a compact message type that contains all necessary information.
  32. """
  33. def __init__(self, msg_id: int, src, dst, type_: MessageType, time: float, refer_msg_id: int=-1, line_no = -1):
  34. """
  35. Constructs a message with the given parameters.
  36. :param msg_id: the ID of the message
  37. :param src: something identifiying the source, e.g. ID or configuration
  38. :param dst: something identifiying the destination, e.g. ID or configuration
  39. :param type_: the type of the message
  40. :param time: the timestamp of the message
  41. :param refer_msg_id: the ID this message is a request for or reply to. -1 if there is no related message.
  42. :param line_no: The line number this message appeared at in the original CSV file
  43. """
  44. self.msg_id = msg_id
  45. self.src = src
  46. self.dst = dst
  47. self.type = type_
  48. self.time = time
  49. self.csv_time = time
  50. self.refer_msg_id = refer_msg_id
  51. self.line_no = line_no
  52. def __str__(self):
  53. str_ = "{0}. at {1}: {2}-->{3}, {4}, refer:{5} (line {6})".format(self.msg_id, self.time, self.src, self.dst, self.type, self.refer_msg_id, self.line_no)
  54. return str_
  55. from ID2TLib import FileUtils, Generator
  56. from ID2TLib.IPv4 import IPAddress
  57. from ID2TLib.PcapAddressOperations import PcapAddressOperations
  58. from ID2TLib.Botnet.CommunicationProcessor import CommunicationProcessor
  59. from ID2TLib.Botnet.MessageMapping import MessageMapping
  60. from ID2TLib.PcapFile import PcapFile
  61. from Core.Statistics import Statistics
  62. from scapy.layers.inet import IP, IPOption_Security
  63. class MembersMgmtCommAttack(BaseAttack.BaseAttack):
  64. def __init__(self):
  65. """
  66. Creates a new instance of the Membership Management Communication.
  67. """
  68. # Initialize communication
  69. super(MembersMgmtCommAttack, self).__init__("Membership Management Communication Attack (MembersMgmtCommAttack)",
  70. "Injects Membership Management Communication", "Botnet communication")
  71. # Define allowed parameters and their type
  72. self.supported_params = {
  73. # parameters regarding attack
  74. Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
  75. Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
  76. Param.PACKETS_LIMIT: ParameterTypes.TYPE_INTEGER_POSITIVE,
  77. Param.ATTACK_DURATION: ParameterTypes.TYPE_INTEGER_POSITIVE,
  78. # use num_attackers to specify number of communicating devices?
  79. Param.NUMBER_INITIATOR_BOTS: ParameterTypes.TYPE_INTEGER_POSITIVE,
  80. # input file containing botnet communication
  81. Param.FILE_CSV: ParameterTypes.TYPE_FILEPATH,
  82. Param.FILE_XML: ParameterTypes.TYPE_FILEPATH,
  83. # the percentage of IP reuse (if total and other is specified, percentages are multiplied)
  84. Param.IP_REUSE_TOTAL: ParameterTypes.TYPE_PERCENTAGE,
  85. Param.IP_REUSE_LOCAL: ParameterTypes.TYPE_PERCENTAGE,
  86. Param.IP_REUSE_EXTERNAL: ParameterTypes.TYPE_PERCENTAGE,
  87. # the user-selected padding to add to every packet
  88. Param.PACKET_PADDING: ParameterTypes.TYPE_PADDING,
  89. # presence of NAT at the gateway of the network
  90. Param.NAT_PRESENT: ParameterTypes.TYPE_BOOLEAN,
  91. # whether the TTL distribution should be based on the input PCAP
  92. # or the CAIDA dataset
  93. Param.TTL_FROM_CAIDA: ParameterTypes.TYPE_BOOLEAN,
  94. # whether the destination port of a response should be the ephemeral port
  95. # its request came from or a static (server)port based on a hostname
  96. Param.MULTIPORT: ParameterTypes.TYPE_BOOLEAN,
  97. # information about the interval selection strategy
  98. Param.INTERVAL_SELECT_STRATEGY: ParameterTypes.TYPE_INTERVAL_SELECT_STRAT,
  99. Param.INTERVAL_SELECT_START: ParameterTypes.TYPE_INTEGER_POSITIVE,
  100. Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE,
  101. # determines whether injected packets are marked with an unused IP option
  102. # to easily filter them in e.g. wireshark
  103. Param.HIDDEN_MARK: ParameterTypes.TYPE_BOOLEAN
  104. }
  105. # create dict with MessageType values for fast name lookup
  106. self.msg_types = {}
  107. for msg_type in MessageType:
  108. self.msg_types[msg_type.value] = msg_type
  109. def init_params(self):
  110. """
  111. Initialize some parameters of this communication-attack using the user supplied command line parameters.
  112. The remaining parameters are implicitly set in the provided data file. Note: the timestamps in the file
  113. have to be sorted in ascending order
  114. :param statistics: Reference to a statistics object.
  115. """
  116. # set class constants
  117. self.DEFAULT_XML_PATH = "resources/MembersMgmtComm_example.xml"
  118. # probability for responder ID to be local if comm_type is mixed
  119. self.PROB_RESPND_IS_LOCAL = 0
  120. # PARAMETERS: initialize with default values
  121. # (values are overwritten if user specifies them)
  122. self.add_param_value(Param.INJECT_AFTER_PACKET, 1 + randint(0, self.statistics.get_packet_count() // 5))
  123. self.add_param_value(Param.FILE_XML, self.DEFAULT_XML_PATH)
  124. # Alternatively new attack parameter?
  125. duration = int(float(self._get_capture_duration()))
  126. self.add_param_value(Param.ATTACK_DURATION, duration)
  127. self.add_param_value(Param.NUMBER_INITIATOR_BOTS, 1)
  128. # NAT on by default
  129. self.add_param_value(Param.NAT_PRESENT, True)
  130. # TODO: change 1 to something better
  131. self.add_param_value(Param.IP_REUSE_TOTAL, 1)
  132. self.add_param_value(Param.IP_REUSE_LOCAL, 0.5)
  133. self.add_param_value(Param.IP_REUSE_EXTERNAL, 0.5)
  134. # add default additional padding
  135. self.add_param_value(Param.PACKET_PADDING, 20)
  136. # choose the input PCAP as default base for the TTL distribution
  137. self.add_param_value(Param.TTL_FROM_CAIDA, False)
  138. # do not use multiple ports for requests and responses
  139. self.add_param_value(Param.MULTIPORT, False)
  140. # interval selection strategy
  141. self.add_param_value(Param.INTERVAL_SELECT_STRATEGY, "optimal")
  142. self.add_param_value(Param.HIDDEN_MARK, False)
  143. def generate_attack_pcap(self):
  144. """
  145. Injects the packets of this attack into a PCAP and stores it as a temporary file.
  146. :return: a tuple of the number packets injected and the path to the temporary attack PCAP
  147. """
  148. # create the final messages that have to be sent, including all bot configurations
  149. messages = self._create_messages()
  150. if messages == []:
  151. return 0, []
  152. # Setup (initial) parameters for packet creation loop
  153. BUFFER_SIZE = 1000
  154. pkt_gen = Generator.PacketGenerator()
  155. padding = self.get_param_value(Param.PACKET_PADDING)
  156. packets = deque(maxlen=BUFFER_SIZE)
  157. total_pkts = 0
  158. limit_packetcount = self.get_param_value(Param.PACKETS_LIMIT)
  159. limit_duration = self.get_param_value(Param.ATTACK_DURATION)
  160. path_attack_pcap = None
  161. overThousand = False
  162. msg_packet_mapping = MessageMapping(messages, self.statistics.get_pcap_timestamp_start())
  163. mark_packets = self.get_param_value(Param.HIDDEN_MARK)
  164. # create packets to write to PCAP file
  165. for msg in messages:
  166. # retrieve the source and destination configurations
  167. id_src, id_dst = msg.src["ID"], msg.dst["ID"]
  168. ip_src, ip_dst = msg.src["IP"], msg.dst["IP"]
  169. mac_src, mac_dst = msg.src["MAC"], msg.dst["MAC"]
  170. if msg.type.is_request():
  171. port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
  172. else:
  173. port_src, port_dst = int(msg.src["DstPort"]), int(msg.dst["SrcPort"])
  174. ttl = int(msg.src["TTL"])
  175. # update duration
  176. duration = msg.time - messages[0].time
  177. # if total number of packets has been sent or the attack duration has been exceeded, stop
  178. if ((limit_packetcount is not None and total_pkts >= limit_packetcount) or
  179. (limit_duration is not None and duration >= limit_duration)):
  180. break
  181. # if the type of the message is a NL reply, determine the number of entries
  182. nl_size = 0
  183. if msg.type == MessageType.SALITY_NL_REPLY:
  184. nl_size = randint(1, 25) # what is max NL entries?
  185. # create suitable IP/UDP packet and add to packets list
  186. packet = pkt_gen.generate_mmcom_packet(ip_src=ip_src, ip_dst=ip_dst, ttl=ttl, mac_src=mac_src, mac_dst=mac_dst,
  187. port_src=port_src, port_dst=port_dst, message_type=msg.type, neighborlist_entries=nl_size)
  188. Generator.add_padding(packet, padding,True, True)
  189. packet.time = msg.time
  190. if mark_packets and isinstance(packet.payload, IP): # do this only for ip-packets
  191. ip_data = packet.payload
  192. hidden_opt = IPOption_Security()
  193. hidden_opt.option = 2 # "normal" security opt
  194. hidden_opt.security = 16 # magic value indicating NSA
  195. ip_data.options = hidden_opt
  196. packets.append(packet)
  197. msg_packet_mapping.map_message(msg, packet)
  198. total_pkts += 1
  199. # Store timestamp of first packet (for attack label)
  200. if total_pkts <= 1:
  201. self.attack_start_utime = packets[0].time
  202. elif total_pkts % BUFFER_SIZE == 0: # every 1000 packets write them to the PCAP file (append)
  203. if overThousand: # if over 1000 packets written, there may be a different packet-length for the last few packets
  204. packets = list(packets)
  205. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  206. last_packet = packets[-1]
  207. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  208. packets = deque(maxlen=BUFFER_SIZE)
  209. else:
  210. packets = list(packets)
  211. Generator.equal_length(packets, padding = padding)
  212. last_packet = packets[-1]
  213. max_len = len(last_packet)
  214. overThousand = True
  215. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  216. packets = deque(maxlen=BUFFER_SIZE)
  217. # if there are unwritten packets remaining, write them to the PCAP file
  218. if len(packets) > 0:
  219. if overThousand:
  220. packets = list(packets)
  221. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  222. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  223. last_packet = packets[-1]
  224. else:
  225. packets = list(packets)
  226. Generator.equal_length(packets, padding = padding)
  227. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  228. last_packet = packets[-1]
  229. # write the mapping to a file
  230. current_ts = datetime.now().strftime("%Y%m%d-%H%M%S")
  231. mapping_filename = "mapping_" + current_ts
  232. msg_packet_mapping.write_to_file(mapping_filename)
  233. Util.MISC_OUT_FILES["mapping.xml"] = mapping_filename
  234. # Store timestamp of last packet
  235. self.attack_end_utime = last_packet.time
  236. # Return packets sorted by packet by timestamp and total number of packets (sent)
  237. return total_pkts , path_attack_pcap
  238. def generate_attack_packets(self):
  239. pass
  240. def _create_messages(self):
  241. """
  242. Creates the messages that are to be injected into the PCAP.
  243. :return: the final messages as a list
  244. """
  245. def add_ids_to_config(ids_to_add: list, existing_ips: list, new_ips: list, bot_configs: dict, idtype:str="local", router_mac:str=""):
  246. """
  247. Creates IP and MAC configurations for the given IDs and adds them to the existing configurations object.
  248. :param ids_to_add: all sorted IDs that have to be configured and added
  249. :param existing_ips: the existing IPs in the PCAP file that should be assigned to some, or all, IDs
  250. :param new_ips: the newly generated IPs that should be assigned to some, or all, IDs
  251. :param bot_configs: the existing configurations for the bots
  252. :param idtype: the locality type of the IDs
  253. :param router_mac: the MAC address of the router in the PCAP
  254. """
  255. ids = ids_to_add.copy()
  256. # macgen only needed, when IPs are new local IPs (therefore creating the object here suffices for the current callers
  257. # to not end up with the same MAC paired with different IPs)
  258. macgen = Generator.MacAddressGenerator()
  259. # assign existing IPs and the corresponding MAC addresses in the PCAP to the IDs
  260. for ip in existing_ips:
  261. random_id = choice(ids)
  262. mac = self.statistics.process_db_query("macAddress(IPAddress=%s)" % ip)
  263. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  264. ids.remove(random_id)
  265. # assign new IPs and for local IPs new MACs or for external IPs the router MAC to the IDs
  266. for ip in new_ips:
  267. random_id = choice(ids)
  268. if idtype == "local":
  269. mac = macgen.random_mac()
  270. elif idtype == "external":
  271. mac = router_mac
  272. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  273. ids.remove(random_id)
  274. def index_increment(number: int, max: int):
  275. """
  276. Number increment with rollover.
  277. """
  278. if number + 1 < max:
  279. return number + 1
  280. else:
  281. return 0
  282. def assign_realistic_ttls(bot_configs:list):
  283. '''
  284. Assigns a realisitic ttl to each bot from @param: bot_configs. Uses statistics and distribution to be able
  285. to calculate a realisitc ttl.
  286. :param bot_configs: List that contains all bots that should be assigned with realistic ttls.
  287. '''
  288. ids = sorted(bot_configs.keys())
  289. for pos,bot in enumerate(ids):
  290. bot_type = bot_configs[bot]["Type"]
  291. # print(bot_type)
  292. if(bot_type == "local"): # Set fix TTL for local Bots
  293. bot_configs[bot]["TTL"] = 128
  294. # Set TTL based on TTL distribution of IP address
  295. else: # Set varying TTl for external Bots
  296. bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
  297. if len(bot_ttl_dist) > 0:
  298. source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
  299. bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
  300. else:
  301. bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
  302. def assign_realistic_timestamps(messages: list, external_ids: set, local_ids: set, avg_delay_local:float, avg_delay_external: float, zero_reference:float):
  303. """
  304. Assigns realistic timestamps to a set of messages
  305. :param messages: the set of messages to be updated
  306. :param external_ids: the set of bot ids, that are outside the network, i.e. external
  307. :param local_ids: the set of bot ids, that are inside the network, i.e. local
  308. :avg_delay_local: the avg_delay between the dispatch and the reception of a packet between local computers
  309. :avg_delay_external: the avg_delay between the dispatch and the reception of a packet between a local and an external computer
  310. :zero_reference: the timestamp which is regarded as the beginning of the pcap_file and therefore handled like a timestamp that resembles 0
  311. """
  312. updated_msgs = []
  313. last_response = {} # Dict, takes a tuple of 2 Bot_IDs as a key (requester, responder), returns the time of the last response, the requester received
  314. # necessary in order to make sure, that additional requests are sent only after the response to the last one was received
  315. for msg in messages: # init
  316. last_response[(msg.src, msg.dst)] = -1
  317. # update all timestamps
  318. for req_msg in messages:
  319. if(req_msg in updated_msgs):
  320. # message already updated
  321. continue
  322. # if req_msg.timestamp would be before the timestamp of the response to the last request, req_msg needs to be sent later (else branch)
  323. if last_response[(req_msg.src, req_msg.dst)] == -1 or last_response[(req_msg.src, req_msg.dst)] < (zero_reference + req_msg.time - 0.05):
  324. ## update req_msg timestamp with a variation of up to 50ms
  325. req_msg.time = zero_reference + req_msg.time + uniform(-0.05, 0.05)
  326. updated_msgs.append(req_msg)
  327. else:
  328. req_msg.time = last_response[(req_msg.src, req_msg.dst)] + 0.06 + uniform(-0.05, 0.05)
  329. # update response if necessary
  330. if req_msg.refer_msg_id != -1:
  331. respns_msg = messages[req_msg.refer_msg_id]
  332. # check for local or external communication and update response timestamp with the respective avg delay
  333. if req_msg.src in external_ids or req_msg.dst in external_ids:
  334. #external communication
  335. respns_msg.time = req_msg.time + avg_delay_external + uniform(-0.1*avg_delay_external, 0.1*avg_delay_external)
  336. else:
  337. #local communication
  338. respns_msg.time = req_msg.time + avg_delay_local + uniform(-0.1*avg_delay_local, 0.1*avg_delay_local)
  339. updated_msgs.append(respns_msg)
  340. last_response[(req_msg.src, req_msg.dst)] = respns_msg.time
  341. def assign_ttls_from_caida(bot_configs):
  342. """
  343. Assign realistic TTL values to bots with respect to their IP, based on the CAIDA dataset.
  344. If there exists an entry for a bot's IP, the TTL is chosen based on a distribution over all used TTLs by this IP.
  345. If there is no such entry, the TTL is chosen based on a distribution over all used TTLs and their respective frequency.
  346. :param bot_configs: the existing bot configurations
  347. """
  348. def get_ip_ttl_distrib():
  349. """
  350. Parses the CSV file containing a mapping between IP and their used TTLs.
  351. :return: returns a dict with the IPs as keys and dicts for their TTL disribution as values
  352. """
  353. ip_based_distrib = {}
  354. with open("resources/CaidaTTL_perIP.csv", "r") as file:
  355. # every line consists of: IP, TTL, Frequency
  356. next(file) # skip CSV header line
  357. for line in file:
  358. ip_addr, ttl, freq = line.split(",")
  359. if ip_addr not in ip_based_distrib:
  360. ip_based_distrib[ip_addr] = {} # the values for ip_based_distrib are dicts with key=TTL, value=Frequency
  361. ip_based_distrib[ip_addr][ttl] = int(freq)
  362. return ip_based_distrib
  363. def get_total_ttl_distrib():
  364. """
  365. Parses the CSV file containing an overview of all used TTLs and their respective frequency.
  366. :return: returns a dict with the TTLs as keys and their frequencies as keys
  367. """
  368. total_ttl_distrib = {}
  369. with open("resources/CaidaTTL_total.csv", "r") as file:
  370. # every line consists of: TTL, Frequency, Fraction
  371. next(file) # skip CSV header line
  372. for line in file:
  373. ttl, freq, _ = line.split(",")
  374. total_ttl_distrib[ttl] = int(freq)
  375. return total_ttl_distrib
  376. # get the TTL distribution for every IP that is available in "resources/CaidaTTL_perIP.csv"
  377. ip_ttl_distrib = get_ip_ttl_distrib()
  378. # build a probability dict for the total TTL distribution
  379. total_ttl_prob_dict = Lea.fromValFreqsDict(get_total_ttl_distrib())
  380. # loop over every bot id and assign a TTL to the respective bot
  381. for bot_id in sorted(bot_configs):
  382. bot_type = bot_configs[bot_id]["Type"]
  383. bot_ip = bot_configs[bot_id]["IP"]
  384. if bot_type == "local":
  385. bot_configs[bot_id]["TTL"] = 128
  386. # if there exists detailed information about the TTL distribution of this IP
  387. elif bot_ip in ip_ttl_distrib:
  388. ip_ttl_freqs = ip_ttl_distrib[bot_ip]
  389. source_ttl_prob_dict = Lea.fromValFreqsDict(ip_ttl_freqs) # build a probability dict from this IP's TTL distribution
  390. bot_configs[bot_id]["TTL"] = source_ttl_prob_dict.random()
  391. # otherwise assign a random TTL based on the total TTL distribution
  392. else:
  393. bot_configs[bot_id]["TTL"] = total_ttl_prob_dict.random()
  394. # parse input CSV or XML
  395. filepath_xml = self.get_param_value(Param.FILE_XML)
  396. filepath_csv = self.get_param_value(Param.FILE_CSV)
  397. # use C++ communication processor for faster interval finding
  398. cpp_comm_proc = lb.botnet_comm_processor();
  399. # only use CSV input if the XML path is the default one
  400. # --> prefer XML input over CSV input (in case both are given)
  401. print_updates = False
  402. if filepath_csv and filepath_xml == self.DEFAULT_XML_PATH:
  403. filepath_xml = OUT_DIR + os.path.splitext(os.path.basename(filepath_csv))[0]
  404. filesize = os.path.getsize(filepath_csv) / 2**20 # get filesize in MB
  405. if filesize > 10:
  406. print("\nParsing input CSV file...", end=" ")
  407. sys.stdout.flush()
  408. print_updates = True
  409. cpp_comm_proc.parse_csv(filepath_csv)
  410. if print_updates:
  411. print("done.")
  412. print("Writing corresponding XML file...", end=" ")
  413. sys.stdout.flush()
  414. filepath_xml = cpp_comm_proc.write_xml(filepath_xml)
  415. Util.MISC_OUT_FILES[filepath_xml] = None
  416. if print_updates: print("done.")
  417. else:
  418. filesize = os.path.getsize(filepath_xml) / 2**20 # get filesize in MB
  419. if filesize > 10:
  420. print("Parsing input XML file...", end=" ")
  421. sys.stdout.flush()
  422. print_updates = True
  423. cpp_comm_proc.parse_xml(filepath_xml)
  424. if print_updates: print("done.")
  425. # find a good communication mapping in the input file that matches the users parameters
  426. nat = self.get_param_value(Param.NAT_PRESENT)
  427. comm_proc = CommunicationProcessor(self.msg_types, nat)
  428. duration = self.get_param_value(Param.ATTACK_DURATION)
  429. number_init_bots = self.get_param_value(Param.NUMBER_INITIATOR_BOTS)
  430. strategy = self.get_param_value(Param.INTERVAL_SELECT_STRATEGY)
  431. start_idx = self.get_param_value(Param.INTERVAL_SELECT_START)
  432. end_idx = self.get_param_value(Param.INTERVAL_SELECT_END)
  433. potential_long_find_time = (strategy == "optimal" and (filesize > 4 and self.statistics.get_packet_count() > 1000))
  434. if print_updates or potential_long_find_time:
  435. if not print_updates: print()
  436. print("Selecting communication interval from input CSV/XML file...", end=" ")
  437. sys.stdout.flush()
  438. if potential_long_find_time:
  439. print("\nWarning: Because of the large input files and the (chosen) interval selection strategy 'optimal',")
  440. print("this may take a while. Consider using selection strategy 'random' or 'custom'...", end=" ")
  441. sys.stdout.flush()
  442. print_updates = True
  443. comm_interval = comm_proc.get_comm_interval(cpp_comm_proc, strategy, number_init_bots, duration, start_idx, end_idx)
  444. if not comm_interval:
  445. print("Error: An interval that satisfies the input cannot be found.")
  446. return []
  447. if print_updates: print("done.") # print corresponding message to interval finding message
  448. # retrieve the mapping information
  449. mapped_ids, packet_start_idx, packet_end_idx = comm_interval["IDs"], comm_interval["Start"], comm_interval["End"]
  450. # print(mapped_ids)
  451. while len(mapped_ids) > number_init_bots:
  452. rm_idx = randrange(0, len(mapped_ids))
  453. del mapped_ids[rm_idx]
  454. if print_updates: print("Generating attack packets...", end=" ")
  455. sys.stdout.flush()
  456. # get the messages contained in the chosen interval
  457. abstract_packets = cpp_comm_proc.get_messages(packet_start_idx, packet_end_idx);
  458. comm_proc.set_mapping(abstract_packets, mapped_ids)
  459. # determine ID roles and select the messages that are to be mapped into the PCAP
  460. messages = comm_proc.det_id_roles_and_msgs()
  461. # use the previously detetermined roles to assign the locality of all IDs
  462. local_ids, external_ids = comm_proc.det_ext_and_local_ids()
  463. # print start and end time of mapped interval
  464. # print(abstract_packets[packet_start_idx]["Time"])
  465. # print(abstract_packets[packet_end_idx]["Time"])
  466. # print(mapped_ids)
  467. # determine number of reused local and external IPs
  468. reuse_percent_total = self.get_param_value(Param.IP_REUSE_TOTAL)
  469. reuse_percent_external = self.get_param_value(Param.IP_REUSE_EXTERNAL)
  470. reuse_percent_local = self.get_param_value(Param.IP_REUSE_LOCAL)
  471. reuse_count_external = int(reuse_percent_total * reuse_percent_external * len(mapped_ids))
  472. reuse_count_local = int(reuse_percent_total * reuse_percent_local * len(mapped_ids))
  473. # create IP and MAC configurations for the IDs/Bots
  474. ipgen = Generator.IPGenerator()
  475. pcapops = PcapAddressOperations(self.statistics)
  476. router_mac = pcapops.get_probable_router_mac()
  477. bot_configs = {}
  478. # retrieve and assign the IPs and MACs for the bots with respect to the given parameters
  479. # (IDs are always added to bot_configs in the same order under a given seed)
  480. number_local_ids, number_external_ids = len(local_ids), len(external_ids)
  481. # assign addresses for local IDs
  482. if number_local_ids > 0:
  483. reuse_count_local = int(reuse_percent_total * reuse_percent_local * number_local_ids)
  484. existing_local_ips = sorted(pcapops.get_existing_local_ips(reuse_count_local))
  485. new_local_ips = sorted(pcapops.get_new_local_ips(number_local_ids - len(existing_local_ips)))
  486. add_ids_to_config(sorted(local_ids), existing_local_ips, new_local_ips, bot_configs)
  487. # assign addresses for external IDs
  488. if number_external_ids > 0:
  489. reuse_count_external = int(reuse_percent_total * reuse_percent_external * number_external_ids)
  490. existing_external_ips = sorted(pcapops.get_existing_external_ips(reuse_count_external))
  491. remaining = len(external_ids) - len(existing_external_ips)
  492. for external_ip in existing_external_ips: ipgen.add_to_blacklist(external_ip)
  493. new_external_ips = sorted([ipgen.random_ip() for _ in range(remaining)])
  494. add_ids_to_config(sorted(external_ids), existing_external_ips, new_external_ips, bot_configs, idtype="external", router_mac=router_mac)
  495. # this is the timestamp at which the first packet should be injected, the packets have to be shifted to the beginning of the
  496. # pcap file (INJECT_AT_TIMESTAMP) and then the offset of the packets have to be compensated to start at the given point in time
  497. zero_reference = self.get_param_value(Param.INJECT_AT_TIMESTAMP) - messages[0].time
  498. # calculate the average delay values for local and external responses
  499. avg_delay_local, avg_delay_external = self.statistics.get_avg_delay_local_ext()
  500. #set timestamps
  501. assign_realistic_timestamps(messages, external_ids, local_ids, avg_delay_local, avg_delay_external, zero_reference)
  502. portSelector = PortSelectors.LINUX
  503. reserved_ports = set(int(line.strip()) for line in open("resources/reserved_ports.txt").readlines())
  504. def filter_reserved(get_port):
  505. port = get_port()
  506. while port in reserved_ports:
  507. port = get_port()
  508. return port
  509. # create port configurations for the bots
  510. use_multiple_ports = self.get_param_value(Param.MULTIPORT)
  511. for bot in sorted(bot_configs):
  512. bot_configs[bot]["SrcPort"] = filter_reserved(portSelector.select_port_udp)
  513. if not use_multiple_ports:
  514. bot_configs[bot]["DstPort"] = filter_reserved(Generator.gen_random_server_port)
  515. else:
  516. bot_configs[bot]["DstPort"] = filter_reserved(portSelector.select_port_udp)
  517. # assign realistic TTL for every bot
  518. if self.get_param_value(Param.TTL_FROM_CAIDA):
  519. assign_ttls_from_caida(bot_configs)
  520. else:
  521. assign_realistic_ttls(bot_configs)
  522. # put together the final messages including the full sender and receiver
  523. # configurations (i.e. IP, MAC, port, ...) for easier later use
  524. final_messages = []
  525. messages = sorted(messages, key=lambda msg: msg.time)
  526. new_id = 0
  527. for msg in messages:
  528. type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]
  529. id_src, id_dst = msg.src, msg.dst
  530. # sort out messages that do not have a suitable locality setting
  531. if type_src == "external" and type_dst == "external":
  532. continue
  533. msg.src, msg.dst = bot_configs[id_src], bot_configs[id_dst]
  534. msg.src["ID"], msg.dst["ID"] = id_src, id_dst
  535. msg.msg_id = new_id
  536. new_id += 1
  537. ### Important here to update refers, if needed later?
  538. final_messages.append(msg)
  539. return final_messages
  540. def _get_capture_duration(self):
  541. """
  542. Returns the duration of the input PCAP (since statistics duration seems to be incorrect)
  543. """
  544. ts_date_format = "%Y-%m-%d %H:%M:%S.%f"
  545. ts_first_date = datetime.strptime(self.statistics.get_pcap_timestamp_start(), ts_date_format)
  546. ts_last_date = datetime.strptime(self.statistics.get_pcap_timestamp_end(), ts_date_format)
  547. diff_date = ts_last_date - ts_first_date
  548. duration = "%d.%d" % (diff_date.total_seconds(), diff_date.microseconds)
  549. return duration