MembersMgmtCommAttack.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672
  1. from enum import Enum
  2. from random import randint, randrange, choice, uniform
  3. from collections import deque
  4. from scipy.stats import gamma
  5. from lea import Lea
  6. from datetime import datetime
  7. import os
  8. import sys
  9. import ID2TLib.libbotnetcomm as lb
  10. from Attack import BaseAttack
  11. from Attack.AttackParameters import Parameter as Param
  12. from Attack.AttackParameters import ParameterTypes
  13. from ID2TLib.Ports import PortSelectors
  14. class MessageType(Enum):
  15. """
  16. Defines possible botnet message types
  17. """
  18. TIMEOUT = 3
  19. SALITY_NL_REQUEST = 101
  20. SALITY_NL_REPLY = 102
  21. SALITY_HELLO = 103
  22. SALITY_HELLO_REPLY = 104
  23. def is_request(mtype):
  24. return mtype in {MessageType.SALITY_HELLO, MessageType.SALITY_NL_REQUEST}
  25. def is_response(mtype):
  26. return mtype in {MessageType.SALITY_HELLO_REPLY, MessageType.SALITY_NL_REPLY}
  27. class Message():
  28. INVALID_LINENO = -1
  29. """
  30. Defines a compact message type that contains all necessary information.
  31. """
  32. def __init__(self, msg_id: int, src, dst, type_: MessageType, time: float, refer_msg_id: int=-1, line_no = -1):
  33. """
  34. Constructs a message with the given parameters.
  35. :param msg_id: the ID of the message
  36. :param src: something identifiying the source, e.g. ID or configuration
  37. :param dst: something identifiying the destination, e.g. ID or configuration
  38. :param type_: the type of the message
  39. :param time: the timestamp of the message
  40. :param refer_msg_id: the ID this message is a request for or reply to. -1 if there is no related message.
  41. :param line_no: The line number this message appeared at in the original CSV file
  42. """
  43. self.msg_id = msg_id
  44. self.src = src
  45. self.dst = dst
  46. self.type = type_
  47. self.time = time
  48. self.csv_time = time
  49. self.refer_msg_id = refer_msg_id
  50. self.line_no = line_no
  51. def __str__(self):
  52. str_ = "{0}. at {1}: {2}-->{3}, {4}, refer:{5} (line {6})".format(self.msg_id, self.time, self.src, self.dst, self.type, self.refer_msg_id, self.line_no)
  53. return str_
  54. from ID2TLib import FileUtils, Generator
  55. from ID2TLib.IPv4 import IPAddress
  56. from ID2TLib.PcapAddressOperations import PcapAddressOperations
  57. from ID2TLib.CommunicationProcessor import CommunicationProcessor
  58. from ID2TLib.Botnet.MessageMapping import MessageMapping
  59. from ID2TLib.PcapFile import PcapFile
  60. from ID2TLib.Statistics import Statistics
  61. from scapy.layers.inet import IP, IPOption_Security
  62. class MembersMgmtCommAttack(BaseAttack.BaseAttack):
  63. def __init__(self):
  64. """
  65. Creates a new instance of the Membership Management Communication.
  66. """
  67. # Initialize communication
  68. super(MembersMgmtCommAttack, self).__init__("Membership Management Communication Attack (MembersMgmtCommAttack)",
  69. "Injects Membership Management Communication", "Botnet communication")
  70. # Define allowed parameters and their type
  71. self.supported_params = {
  72. # parameters regarding attack
  73. Param.INJECT_AT_TIMESTAMP: ParameterTypes.TYPE_FLOAT,
  74. Param.INJECT_AFTER_PACKET: ParameterTypes.TYPE_PACKET_POSITION,
  75. Param.PACKETS_PER_SECOND: ParameterTypes.TYPE_FLOAT,
  76. Param.PACKETS_LIMIT: ParameterTypes.TYPE_INTEGER_POSITIVE,
  77. Param.ATTACK_DURATION: ParameterTypes.TYPE_INTEGER_POSITIVE,
  78. # use num_attackers to specify number of communicating devices?
  79. Param.NUMBER_INITIATOR_BOTS: ParameterTypes.TYPE_INTEGER_POSITIVE,
  80. # input file containing botnet communication
  81. Param.FILE_CSV: ParameterTypes.TYPE_FILEPATH,
  82. Param.FILE_XML: ParameterTypes.TYPE_FILEPATH,
  83. # the percentage of IP reuse (if total and other is specified, percentages are multiplied)
  84. Param.IP_REUSE_TOTAL: ParameterTypes.TYPE_PERCENTAGE,
  85. Param.IP_REUSE_LOCAL: ParameterTypes.TYPE_PERCENTAGE,
  86. Param.IP_REUSE_EXTERNAL: ParameterTypes.TYPE_PERCENTAGE,
  87. # the user-selected padding to add to every packet
  88. Param.PACKET_PADDING: ParameterTypes.TYPE_PADDING,
  89. # presence of NAT at the gateway of the network
  90. Param.NAT_PRESENT: ParameterTypes.TYPE_BOOLEAN,
  91. # whether the TTL distribution should be based on the input PCAP
  92. # or the CAIDA dataset
  93. Param.TTL_FROM_CAIDA: ParameterTypes.TYPE_BOOLEAN,
  94. # whether the destination port of a response should be the ephemeral port
  95. # its request came from or a static (server)port based on a hostname
  96. Param.MULTIPORT: ParameterTypes.TYPE_BOOLEAN,
  97. # information about the interval selection strategy
  98. Param.INTERVAL_SELECT_STRATEGY: ParameterTypes.TYPE_INTERVAL_SELECT_STRAT,
  99. Param.INTERVAL_SELECT_START: ParameterTypes.TYPE_INTEGER_POSITIVE,
  100. Param.INTERVAL_SELECT_END: ParameterTypes.TYPE_INTEGER_POSITIVE,
  101. Param.HIDDEN_MARK: ParameterTypes.TYPE_BOOLEAN
  102. }
  103. # create dict with MessageType values for fast name lookup
  104. self.msg_types = {}
  105. for msg_type in MessageType:
  106. self.msg_types[msg_type.value] = msg_type
  107. def init_params(self):
  108. """
  109. Initialize some parameters of this communication-attack using the user supplied command line parameters.
  110. The remaining parameters are implicitly set in the provided data file. Note: the timestamps in the file
  111. have to be sorted in ascending order
  112. :param statistics: Reference to a statistics object.
  113. """
  114. # set class constants
  115. self.DEFAULT_XML_PATH = "resources/MembersMgmtComm_example.xml"
  116. # probability for responder ID to be local if comm_type is mixed
  117. self.PROB_RESPND_IS_LOCAL = 0
  118. # PARAMETERS: initialize with default values
  119. # (values are overwritten if user specifies them)
  120. self.add_param_value(Param.INJECT_AFTER_PACKET, 1 + randint(0, self.statistics.get_packet_count() // 5))
  121. self.add_param_value(Param.PACKETS_PER_SECOND, 0)
  122. self.add_param_value(Param.FILE_XML, self.DEFAULT_XML_PATH)
  123. # Alternatively new attack parameter?
  124. duration = int(float(self._get_capture_duration()))
  125. self.add_param_value(Param.ATTACK_DURATION, duration)
  126. self.add_param_value(Param.NUMBER_INITIATOR_BOTS, 1)
  127. # NAT on by default
  128. self.add_param_value(Param.NAT_PRESENT, True)
  129. # TODO: change 1 to something better
  130. self.add_param_value(Param.IP_REUSE_TOTAL, 1)
  131. self.add_param_value(Param.IP_REUSE_LOCAL, 0.5)
  132. self.add_param_value(Param.IP_REUSE_EXTERNAL, 0.5)
  133. # add default additional padding
  134. self.add_param_value(Param.PACKET_PADDING, 20)
  135. # choose the input PCAP as default base for the TTL distribution
  136. self.add_param_value(Param.TTL_FROM_CAIDA, False)
  137. # do not use multiple ports for requests and responses
  138. self.add_param_value(Param.MULTIPORT, False)
  139. # interval selection strategy
  140. self.add_param_value(Param.INTERVAL_SELECT_STRATEGY, "optimal")
  141. self.add_param_value(Param.HIDDEN_MARK, False)
  142. def generate_attack_pcap(self, context):
  143. """
  144. Injects the packets of this attack into a PCAP and stores it as a temporary file.
  145. :param context: the context of the attack, containing e.g. files that are to be created
  146. :return: a tuple of the number packets injected and the path to the temporary attack PCAP
  147. """
  148. # create the final messages that have to be sent, including all bot configurations
  149. messages = self._create_messages(context)
  150. if messages == []:
  151. return 0, []
  152. # Setup (initial) parameters for packet creation loop
  153. BUFFER_SIZE = 1000
  154. pkt_gen = Generator.PacketGenerator()
  155. padding = self.get_param_value(Param.PACKET_PADDING)
  156. packets = deque(maxlen=BUFFER_SIZE)
  157. total_pkts = 0
  158. limit_packetcount = self.get_param_value(Param.PACKETS_LIMIT)
  159. limit_duration = self.get_param_value(Param.ATTACK_DURATION)
  160. path_attack_pcap = None
  161. overThousand = False
  162. msg_packet_mapping = MessageMapping(messages, self.statistics.get_pcap_timestamp_start())
  163. # create packets to write to PCAP file
  164. for msg in messages:
  165. # retrieve the source and destination configurations
  166. id_src, id_dst = msg.src["ID"], msg.dst["ID"]
  167. ip_src, ip_dst = msg.src["IP"], msg.dst["IP"]
  168. mac_src, mac_dst = msg.src["MAC"], msg.dst["MAC"]
  169. if msg.type.is_request():
  170. port_src, port_dst = int(msg.src["SrcPort"]), int(msg.dst["DstPort"])
  171. else:
  172. port_src, port_dst = int(msg.src["DstPort"]), int(msg.dst["SrcPort"])
  173. ttl = int(msg.src["TTL"])
  174. # update duration
  175. duration = msg.time - messages[0].time
  176. # if total number of packets has been sent or the attack duration has been exceeded, stop
  177. if ((limit_packetcount is not None and total_pkts >= limit_packetcount) or
  178. (limit_duration is not None and duration >= limit_duration)):
  179. break
  180. # if the type of the message is a NL reply, determine the number of entries
  181. nl_size = 0
  182. if msg.type == MessageType.SALITY_NL_REPLY:
  183. nl_size = randint(1, 25) # what is max NL entries?
  184. # create suitable IP/UDP packet and add to packets list
  185. packet = pkt_gen.generate_mmcom_packet(ip_src=ip_src, ip_dst=ip_dst, ttl=ttl, mac_src=mac_src, mac_dst=mac_dst,
  186. port_src=port_src, port_dst=port_dst, message_type=msg.type, neighborlist_entries=nl_size)
  187. Generator.add_padding(packet, padding,True, True)
  188. packet.time = msg.time
  189. packets.append(packet)
  190. msg_packet_mapping.map_message(msg, packet)
  191. total_pkts += 1
  192. # Store timestamp of first packet (for attack label)
  193. if total_pkts <= 1:
  194. self.attack_start_utime = packets[0].time
  195. elif total_pkts % BUFFER_SIZE == 0: # every 1000 packets write them to the PCAP file (append)
  196. if overThousand: # if over 1000 packets written, there may be a different packet-length for the last few packets
  197. packets = list(packets)
  198. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  199. last_packet = packets[-1]
  200. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  201. packets = deque(maxlen=BUFFER_SIZE)
  202. else:
  203. packets = list(packets)
  204. Generator.equal_length(packets, padding = padding)
  205. last_packet = packets[-1]
  206. max_len = len(last_packet)
  207. overThousand = True
  208. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  209. packets = deque(maxlen=BUFFER_SIZE)
  210. # if there are unwritten packets remaining, write them to the PCAP file
  211. if len(packets) > 0:
  212. if overThousand:
  213. packets = list(packets)
  214. Generator.equal_length(packets, length = max_len, padding = padding, force_len = True)
  215. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  216. last_packet = packets[-1]
  217. else:
  218. packets = list(packets)
  219. Generator.equal_length(packets, padding = padding)
  220. path_attack_pcap = self.write_attack_pcap(packets, True, path_attack_pcap)
  221. last_packet = packets[-1]
  222. # write the mapping to a file
  223. msg_packet_mapping.write_to(context.allocate_file("_mapping.xml"))
  224. # Store timestamp of last packet
  225. self.attack_end_utime = last_packet.time
  226. if self.get_param_value(Param.HIDDEN_MARK):
  227. # insert an unused ip-option
  228. for p in total_pkts:
  229. if isinstance(p.payload, IP): # do this only for ip-packets
  230. ip_data = p.payload
  231. hidden_opt = IPOption_Security()
  232. hidden_opt.option = 2 # "normal" security opt
  233. hidden_opt.compartment = 16 # magic value indicating NSA
  234. ip_data.options = hidden_opt
  235. # Return packets sorted by packet by timestamp and total number of packets (sent)
  236. return total_pkts , path_attack_pcap
  237. def _create_messages(self, context):
  238. """
  239. Creates the messages that are to be injected into the PCAP.
  240. :param context: the context of the attack, containing e.g. files that are to be created
  241. :return: the final messages as a list
  242. """
  243. def add_ids_to_config(ids_to_add: list, existing_ips: list, new_ips: list, bot_configs: dict, idtype:str="local", router_mac:str=""):
  244. """
  245. Creates IP and MAC configurations for the given IDs and adds them to the existing configurations object.
  246. :param ids_to_add: all sorted IDs that have to be configured and added
  247. :param existing_ips: the existing IPs in the PCAP file that should be assigned to some, or all, IDs
  248. :param new_ips: the newly generated IPs that should be assigned to some, or all, IDs
  249. :param bot_configs: the existing configurations for the bots
  250. :param idtype: the locality type of the IDs
  251. :param router_mac: the MAC address of the router in the PCAP
  252. """
  253. ids = ids_to_add.copy()
  254. # macgen only needed, when IPs are new local IPs (therefore creating the object here suffices for the current callers
  255. # to not end up with the same MAC paired with different IPs)
  256. macgen = Generator.MacAddressGenerator()
  257. # assign existing IPs and the corresponding MAC addresses in the PCAP to the IDs
  258. for ip in existing_ips:
  259. random_id = choice(ids)
  260. mac = self.statistics.process_db_query("macAddress(IPAddress=%s)" % ip)
  261. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  262. ids.remove(random_id)
  263. # assign new IPs and for local IPs new MACs or for external IPs the router MAC to the IDs
  264. for ip in new_ips:
  265. random_id = choice(ids)
  266. if idtype == "local":
  267. mac = macgen.random_mac()
  268. elif idtype == "external":
  269. mac = router_mac
  270. bot_configs[random_id] = {"Type": idtype, "IP": ip, "MAC": mac}
  271. ids.remove(random_id)
  272. def index_increment(number: int, max: int):
  273. """
  274. Number increment with rollover.
  275. """
  276. if number + 1 < max:
  277. return number + 1
  278. else:
  279. return 0
  280. def assign_realistic_ttls(bot_configs:list):
  281. '''
  282. Assigns a realisitic ttl to each bot from @param: bot_configs. Uses statistics and distribution to be able
  283. to calculate a realisitc ttl.
  284. :param bot_configs: List that contains all bots that should be assigned with realistic ttls.
  285. '''
  286. ids = sorted(bot_configs.keys())
  287. for pos,bot in enumerate(ids):
  288. bot_type = bot_configs[bot]["Type"]
  289. # print(bot_type)
  290. if(bot_type == "local"): # Set fix TTL for local Bots
  291. bot_configs[bot]["TTL"] = 128
  292. # Set TTL based on TTL distribution of IP address
  293. else: # Set varying TTl for external Bots
  294. bot_ttl_dist = self.statistics.get_ttl_distribution(bot_configs[bot]["IP"])
  295. if len(bot_ttl_dist) > 0:
  296. source_ttl_prob_dict = Lea.fromValFreqsDict(bot_ttl_dist)
  297. bot_configs[bot]["TTL"] = source_ttl_prob_dict.random()
  298. else:
  299. bot_configs[bot]["TTL"] = self.statistics.process_db_query("most_used(ttlValue)")
  300. def assign_realistic_timestamps(messages: list, external_ids: set, local_ids: set, avg_delay_local:float, avg_delay_external: float, zero_reference:float):
  301. """
  302. Assigns realistic timestamps to a set of messages
  303. :param messages: the set of messages to be updated
  304. :param external_ids: the set of bot ids, that are outside the network, i.e. external
  305. :param local_ids: the set of bot ids, that are inside the network, i.e. local
  306. :avg_delay_local: the avg_delay between the dispatch and the reception of a packet between local computers
  307. :avg_delay_external: the avg_delay between the dispatch and the reception of a packet between a local and an external computer
  308. :zero_reference: the timestamp which is regarded as the beginning of the pcap_file and therefore handled like a timestamp that resembles 0
  309. """
  310. updated_msgs = []
  311. last_response = {} # Dict, takes a tuple of 2 Bot_IDs as a key (requester, responder), returns the time of the last response, the requester received
  312. # necessary in order to make sure, that additional requests are sent only after the response to the last one was received
  313. for msg in messages: # init
  314. last_response[(msg.src, msg.dst)] = -1
  315. # update all timestamps
  316. for req_msg in messages:
  317. if(req_msg in updated_msgs):
  318. # message already updated
  319. continue
  320. # if req_msg.timestamp would be before the timestamp of the response to the last request, req_msg needs to be sent later (else branch)
  321. if last_response[(req_msg.src, req_msg.dst)] == -1 or last_response[(req_msg.src, req_msg.dst)] < (zero_reference + req_msg.time - 0.05):
  322. ## update req_msg timestamp with a variation of up to 50ms
  323. req_msg.time = zero_reference + req_msg.time + uniform(-0.05, 0.05)
  324. updated_msgs.append(req_msg)
  325. else:
  326. req_msg.time = last_response[(req_msg.src, req_msg.dst)] + 0.06 + uniform(-0.05, 0.05)
  327. # update response if necessary
  328. if req_msg.refer_msg_id != -1:
  329. respns_msg = messages[req_msg.refer_msg_id]
  330. # check for local or external communication and update response timestamp with the respective avg delay
  331. if req_msg.src in external_ids or req_msg.dst in external_ids:
  332. #external communication
  333. respns_msg.time = req_msg.time + avg_delay_external + uniform(-0.1*avg_delay_external, 0.1*avg_delay_external)
  334. else:
  335. #local communication
  336. respns_msg.time = req_msg.time + avg_delay_local + uniform(-0.1*avg_delay_local, 0.1*avg_delay_local)
  337. updated_msgs.append(respns_msg)
  338. last_response[(req_msg.src, req_msg.dst)] = respns_msg.time
  339. def assign_ttls_from_caida(bot_configs):
  340. """
  341. Assign realistic TTL values to bots with respect to their IP, based on the CAIDA dataset.
  342. If there exists an entry for a bot's IP, the TTL is chosen based on a distribution over all used TTLs by this IP.
  343. If there is no such entry, the TTL is chosen based on a distribution over all used TTLs and their respective frequency.
  344. :param bot_configs: the existing bot configurations
  345. """
  346. def get_ip_ttl_distrib():
  347. """
  348. Parses the CSV file containing a mapping between IP and their used TTLs.
  349. :return: returns a dict with the IPs as keys and dicts for their TTL disribution as values
  350. """
  351. ip_based_distrib = {}
  352. with open("resources/CaidaTTL_perIP.csv", "r") as file:
  353. # every line consists of: IP, TTL, Frequency
  354. next(file) # skip CSV header line
  355. for line in file:
  356. ip_addr, ttl, freq = line.split(",")
  357. if ip_addr not in ip_based_distrib:
  358. ip_based_distrib[ip_addr] = {} # the values for ip_based_distrib are dicts with key=TTL, value=Frequency
  359. ip_based_distrib[ip_addr][ttl] = int(freq)
  360. return ip_based_distrib
  361. def get_total_ttl_distrib():
  362. """
  363. Parses the CSV file containing an overview of all used TTLs and their respective frequency.
  364. :return: returns a dict with the TTLs as keys and their frequencies as keys
  365. """
  366. total_ttl_distrib = {}
  367. with open("resources/CaidaTTL_total.csv", "r") as file:
  368. # every line consists of: TTL, Frequency, Fraction
  369. next(file) # skip CSV header line
  370. for line in file:
  371. ttl, freq, _ = line.split(",")
  372. total_ttl_distrib[ttl] = int(freq)
  373. return total_ttl_distrib
  374. # get the TTL distribution for every IP that is available in "resources/CaidaTTL_perIP.csv"
  375. ip_ttl_distrib = get_ip_ttl_distrib()
  376. # build a probability dict for the total TTL distribution
  377. total_ttl_prob_dict = Lea.fromValFreqsDict(get_total_ttl_distrib())
  378. # loop over every bot id and assign a TTL to the respective bot
  379. for bot_id in sorted(bot_configs):
  380. bot_type = bot_configs[bot_id]["Type"]
  381. bot_ip = bot_configs[bot_id]["IP"]
  382. if bot_type == "local":
  383. bot_configs[bot_id]["TTL"] = 128
  384. # if there exists detailed information about the TTL distribution of this IP
  385. elif bot_ip in ip_ttl_distrib:
  386. ip_ttl_freqs = ip_ttl_distrib[bot_ip]
  387. source_ttl_prob_dict = Lea.fromValFreqsDict(ip_ttl_freqs) # build a probability dict from this IP's TTL distribution
  388. bot_configs[bot_id]["TTL"] = source_ttl_prob_dict.random()
  389. # otherwise assign a random TTL based on the total TTL distribution
  390. else:
  391. bot_configs[bot_id]["TTL"] = total_ttl_prob_dict.random()
  392. def move_xml_to_outdir(filepath_xml: str):
  393. """
  394. Moves the XML file at filepath_xml to the output directory of the PCAP
  395. :param filepath_xml: the filepath to the XML file
  396. :return: the new filepath to the XML file
  397. """
  398. pcap_dir = context.get_output_dir()
  399. xml_name = os.path.basename(filepath_xml)
  400. if pcap_dir.endswith("/"):
  401. new_xml_path = pcap_dir + xml_name
  402. else:
  403. new_xml_path = pcap_dir + "/" + xml_name
  404. os.rename(filepath_xml, new_xml_path)
  405. context.add_other_created_file(new_xml_path)
  406. return new_xml_path
  407. # parse input CSV or XML
  408. filepath_xml = self.get_param_value(Param.FILE_XML)
  409. filepath_csv = self.get_param_value(Param.FILE_CSV)
  410. # use C++ communication processor for faster interval finding
  411. cpp_comm_proc = lb.botnet_comm_processor();
  412. # only use CSV input if the XML path is the default one
  413. # --> prefer XML input over CSV input (in case both are given)
  414. print_updates = False
  415. if filepath_csv and filepath_xml == self.DEFAULT_XML_PATH:
  416. filename = os.path.splitext(filepath_csv)[0]
  417. filesize = os.path.getsize(filepath_csv) / 2**20 # get filesize in MB
  418. if filesize > 10:
  419. print("\nParsing input CSV file...", end=" ")
  420. sys.stdout.flush()
  421. print_updates = True
  422. cpp_comm_proc.parse_csv(filepath_csv)
  423. if print_updates:
  424. print("done.")
  425. print("Writing corresponding XML file...", end=" ")
  426. sys.stdout.flush()
  427. filepath_xml = cpp_comm_proc.write_xml(filename)
  428. filepath_xml = move_xml_to_outdir(filepath_xml)
  429. if print_updates: print("done.")
  430. else:
  431. filesize = os.path.getsize(filepath_xml) / 2**20 # get filesize in MB
  432. if filesize > 10:
  433. print("Parsing input XML file...", end=" ")
  434. sys.stdout.flush()
  435. print_updates = True
  436. cpp_comm_proc.parse_xml(filepath_xml)
  437. if print_updates: print("done.")
  438. # find a good communication mapping in the input file that matches the users parameters
  439. nat = self.get_param_value(Param.NAT_PRESENT)
  440. comm_proc = CommunicationProcessor(self.msg_types, nat)
  441. duration = self.get_param_value(Param.ATTACK_DURATION)
  442. number_init_bots = self.get_param_value(Param.NUMBER_INITIATOR_BOTS)
  443. strategy = self.get_param_value(Param.INTERVAL_SELECT_STRATEGY)
  444. start_idx = self.get_param_value(Param.INTERVAL_SELECT_START)
  445. end_idx = self.get_param_value(Param.INTERVAL_SELECT_END)
  446. potential_long_find_time = (strategy == "optimal" and (filesize > 4 and self.statistics.get_packet_count() > 1000))
  447. if print_updates or potential_long_find_time:
  448. if not print_updates: print()
  449. print("Selecting communication interval from input CSV/XML file...", end=" ")
  450. sys.stdout.flush()
  451. if potential_long_find_time:
  452. print("\nWarning: Because of the large input files and the (chosen) interval selection strategy 'optimal',")
  453. print("this may take a while. Consider using selection strategy 'random' or 'custom'...", end=" ")
  454. sys.stdout.flush()
  455. print_updates = True
  456. comm_interval = comm_proc.get_comm_interval(cpp_comm_proc, strategy, number_init_bots, duration, start_idx, end_idx)
  457. if not comm_interval:
  458. print("Error: An interval that satisfies the input cannot be found.")
  459. return []
  460. if print_updates: print("done.") # print corresponding message to interval finding message
  461. # retrieve the mapping information
  462. mapped_ids, packet_start_idx, packet_end_idx = comm_interval["IDs"], comm_interval["Start"], comm_interval["End"]
  463. # print(mapped_ids)
  464. while len(mapped_ids) > number_init_bots:
  465. rm_idx = randrange(0, len(mapped_ids))
  466. del mapped_ids[rm_idx]
  467. if print_updates: print("Generating attack packets...", end=" ")
  468. sys.stdout.flush()
  469. # get the messages contained in the chosen interval
  470. abstract_packets = cpp_comm_proc.get_messages(packet_start_idx, packet_end_idx);
  471. comm_proc.set_mapping(abstract_packets, mapped_ids)
  472. # determine ID roles and select the messages that are to be mapped into the PCAP
  473. messages = comm_proc.det_id_roles_and_msgs()
  474. # use the previously detetermined roles to assign the locality of all IDs
  475. local_ids, external_ids = comm_proc.det_ext_and_local_ids()
  476. # print start and end time of mapped interval
  477. # print(abstract_packets[packet_start_idx]["Time"])
  478. # print(abstract_packets[packet_end_idx]["Time"])
  479. # print(mapped_ids)
  480. # determine number of reused local and external IPs
  481. reuse_percent_total = self.get_param_value(Param.IP_REUSE_TOTAL)
  482. reuse_percent_external = self.get_param_value(Param.IP_REUSE_EXTERNAL)
  483. reuse_percent_local = self.get_param_value(Param.IP_REUSE_LOCAL)
  484. reuse_count_external = int(reuse_percent_total * reuse_percent_external * len(mapped_ids))
  485. reuse_count_local = int(reuse_percent_total * reuse_percent_local * len(mapped_ids))
  486. # create IP and MAC configurations for the IDs/Bots
  487. ipgen = Generator.IPGenerator()
  488. pcapops = PcapAddressOperations(self.statistics)
  489. router_mac = pcapops.get_probable_router_mac()
  490. bot_configs = {}
  491. # retrieve and assign the IPs and MACs for the bots with respect to the given parameters
  492. # (IDs are always added to bot_configs in the same order under a given seed)
  493. number_local_ids, number_external_ids = len(local_ids), len(external_ids)
  494. # assign addresses for local IDs
  495. if number_local_ids > 0:
  496. reuse_count_local = int(reuse_percent_total * reuse_percent_local * number_local_ids)
  497. existing_local_ips = sorted(pcapops.get_existing_local_ips(reuse_count_local))
  498. new_local_ips = sorted(pcapops.get_new_local_ips(number_local_ids - len(existing_local_ips)))
  499. add_ids_to_config(sorted(local_ids), existing_local_ips, new_local_ips, bot_configs)
  500. # assign addresses for external IDs
  501. if number_external_ids > 0:
  502. reuse_count_external = int(reuse_percent_total * reuse_percent_external * number_external_ids)
  503. existing_external_ips = sorted(pcapops.get_existing_external_ips(reuse_count_external))
  504. remaining = len(external_ids) - len(existing_external_ips)
  505. for external_ip in existing_external_ips: ipgen.add_to_blacklist(external_ip)
  506. new_external_ips = sorted([ipgen.random_ip() for _ in range(remaining)])
  507. add_ids_to_config(sorted(external_ids), existing_external_ips, new_external_ips, bot_configs, idtype="external", router_mac=router_mac)
  508. # this is the timestamp at which the first packet should be injected, the packets have to be shifted to the beginning of the
  509. # pcap file (INJECT_AT_TIMESTAMP) and then the offset of the packets have to be compensated to start at the given point in time
  510. zero_reference = self.get_param_value(Param.INJECT_AT_TIMESTAMP) - messages[0].time
  511. # calculate the average delay values for local and external responses
  512. avg_delay_local, avg_delay_external = self.statistics.get_avg_delay_local_ext()
  513. #set timestamps
  514. assign_realistic_timestamps(messages, external_ids, local_ids, avg_delay_local, avg_delay_external, zero_reference)
  515. portSelector = PortSelectors.LINUX
  516. # create port configurations for the bots
  517. use_multiple_ports = self.get_param_value(Param.MULTIPORT)
  518. for bot in sorted(bot_configs):
  519. bot_configs[bot]["SrcPort"] = portSelector.select_port_udp()
  520. if not use_multiple_ports:
  521. bot_configs[bot]["DstPort"] = Generator.gen_random_server_port()
  522. else:
  523. bot_configs[bot]["DstPort"] = portSelector.select_port_udp()
  524. # assign realistic TTL for every bot
  525. if self.get_param_value(Param.TTL_FROM_CAIDA):
  526. assign_ttls_from_caida(bot_configs)
  527. else:
  528. assign_realistic_ttls(bot_configs)
  529. # put together the final messages including the full sender and receiver
  530. # configurations (i.e. IP, MAC, port, ...) for easier later use
  531. final_messages = []
  532. messages = sorted(messages, key=lambda msg: msg.time)
  533. new_id = 0
  534. for msg in messages:
  535. type_src, type_dst = bot_configs[msg.src]["Type"], bot_configs[msg.dst]["Type"]
  536. id_src, id_dst = msg.src, msg.dst
  537. # sort out messages that do not have a suitable locality setting
  538. if type_src == "external" and type_dst == "external":
  539. continue
  540. msg.src, msg.dst = bot_configs[id_src], bot_configs[id_dst]
  541. msg.src["ID"], msg.dst["ID"] = id_src, id_dst
  542. msg.msg_id = new_id
  543. new_id += 1
  544. ### Important here to update refers, if needed later?
  545. final_messages.append(msg)
  546. return final_messages
  547. def _get_capture_duration(self):
  548. """
  549. Returns the duration of the input PCAP (since statistics duration seems to be incorrect)
  550. """
  551. ts_date_format = "%Y-%m-%d %H:%M:%S.%f"
  552. ts_first_date = datetime.strptime(self.statistics.get_pcap_timestamp_start(), ts_date_format)
  553. ts_last_date = datetime.strptime(self.statistics.get_pcap_timestamp_end(), ts_date_format)
  554. diff_date = ts_last_date - ts_first_date
  555. duration = "%d.%d" % (diff_date.total_seconds(), diff_date.microseconds)
  556. return duration