Merge branch 'utc_timestamps' of stefan.schmidt/ID2T-toolkit into master

code/Core/LabelManager.py

@@ -2,6 +2,7 @@ import importlib
 import datetime as dt
 import os.path
 import xml.dom.minidom as minidom
+import pytz as pytz
 
 import ID2TLib.Label as Label
 import ID2TLib.TestLibrary as Lib
@@ -105,7 +106,7 @@ class LabelManager:
 
             # add timestamp in human-readable format
             timestamp_hr = doc.createElement(self.TAG_TIMESTAMP_HR)
-            timestamp_hr_text = dt.datetime.fromtimestamp(timestamp_entry).strftime('%Y-%m-%d %H:%M:%S.%f')
+            timestamp_hr_text = dt.datetime.fromtimestamp(timestamp_entry).astimezone(pytz.timezone('UTC')).strftime('%Y-%m-%d %H:%M:%S.%f')
             timestamp_hr.appendChild(doc.createTextNode(timestamp_hr_text))
             timestamp_root.appendChild(timestamp_hr)
 

code/ID2TLib/Utility.py

@@ -7,6 +7,7 @@ import lea
 import xdg.BaseDirectory as BaseDir
 import scapy.layers.inet as inet
 import scipy.stats as stats
+import pytz as pytz
 
 CACHE_DIR = os.path.join(BaseDir.xdg_cache_home, 'id2t')
 CODE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/../"
@@ -72,7 +73,7 @@ def update_timestamp(timestamp: float, pps: float, delay: float=0, inj_pps: floa
 
 
 def get_timestamp_from_datetime_str(time: str):
-    return dt.datetime.strptime(time, "%Y-%m-%d %H:%M:%S.%f").timestamp()
+    return pytz.timezone('UTC').localize(dt.datetime.strptime(time, "%Y-%m-%d %H:%M:%S.%f")).timestamp()
 
 
 def get_interval_pps(complement_interval_pps, timestamp):
@@ -189,9 +190,7 @@ def get_filetime_format(timestamp):
     :param timestamp: a timestamp in seconds
     :return: MS FILETIME timestamp
     """
-    boot_datetime = dt.datetime.fromtimestamp(timestamp)
-    if boot_datetime.tzinfo is None or boot_datetime.tzinfo.utcoffset(boot_datetime) is None:
-        boot_datetime = boot_datetime.replace(tzinfo=boot_datetime.tzname())
+    boot_datetime = dt.datetime.fromtimestamp(timestamp).astimezone(pytz.timezone('UTC'))
     boot_filetime = 116444736000000000 + (cal.timegm(boot_datetime.timetuple()) * 10000000)
     return boot_filetime + (boot_datetime.microsecond * 10)
 

code/Test/test_SMBScanAttack.py

@@ -7,7 +7,7 @@ sha_dest_mac_only = 'c42a1775db981a139abd42d031273805cbebd2316b0d8c097217c12193f
 sha_multiple_victims_macos = 'b9a9f423d4154bc38723214124ad74dfdd07a39753563d21f5b453a8c069914a'
 sha_one_victim_linux = '3bb17444446334cf4feee9dd7cbeabd17acbb5ef48525fb3963591f30c37d17a'
 sha_port_shuffle = '08bdecc68fa1a2d1b0dd9802d7d025d42d90b9184d1fb6e1bcab234fac7db1b4'
-sha_smb2 = 'ef525fb61612a3db90bd0bbfaf4412f682933954357c941e1d8ac05c9ec373d4'
+sha_smb2 = '315bc052fd045f8738021062e8b5f77a33c649adfed490d3c9da94c97ba32f95'
 sha_ip_src_shuffle = '1d699ca109c62000b77b53002f1087ebf5ccc2c2dead1dbc5c18b5f6311273d0'
 sha_victim_range_winxp_hosting = 'bd624da4e3b7a3f06b8154ed9d6274d498b589aaaa11c2d0dc207a80ab7205b9'
 

code/Test/test_internalQueries.py

@@ -22,10 +22,10 @@ class UnitTestInternalQueries(unittest.TestCase):
                           ("Recognized packets", 1988, "packets"),
                           ("Unrecognized packets", 10, "PDUs"), ("% Recognized packets", 99.49949949949949, "%"),
                           ("% Unrecognized packets", 0.5005005005005005, "%"),
-                          ("Last unknown PDU", '1970-01-01 01:07:39.604899'),
+                          ("Last unknown PDU", '1970-01-01 00:07:39.604899'),
                           ('Capture duration', '384.454345703125', 'seconds'),
-                          ('Capture start', '\t1970-01-01 01:01:45.647675'),
-                          ('Capture end', '\t1970-01-01 01:08:10.102034')])
+                          ('Capture start', '\t1970-01-01 00:01:45.647675'),
+                          ('Capture end', '\t1970-01-01 00:08:10.102034')])
 
     def test_get_packet_count(self):
         self.assertEqual(controller.statistics.get_packet_count(), 1998)
@@ -34,10 +34,10 @@ class UnitTestInternalQueries(unittest.TestCase):
         self.assertEqual(controller.statistics.get_capture_duration(), '384.454345703125')
 
     def test_get_pcap_timestamp_start(self):
-        self.assertEqual(controller.statistics.get_pcap_timestamp_start(), '1970-01-01 01:01:45.647675')
+        self.assertEqual(controller.statistics.get_pcap_timestamp_start(), '1970-01-01 00:01:45.647675')
 
     def test_get_pcap_timestamp_end(self):
-        self.assertEqual(controller.statistics.get_pcap_timestamp_end(), '1970-01-01 01:08:10.102034')
+        self.assertEqual(controller.statistics.get_pcap_timestamp_end(), '1970-01-01 00:08:10.102034')
 
     # FIXME: This seems to be the only testcase where float values differ slightly between macOS and Linux
     def test_get_general_file_statistics(self):

code_boost/src/cxx/statistics.cpp

@@ -667,7 +667,7 @@ std::string statistics::getFormattedTimestamp(time_t seconds, suseconds_t micros
     tv.tv_sec = seconds;
     tv.tv_usec = microseconds;
     char tmbuf[64], buf[64];
-    auto nowtm = localtime(&(tv.tv_sec));
+    auto nowtm = gmtime(&(tv.tv_sec));
     strftime(tmbuf, sizeof(tmbuf), "%Y-%m-%d %H:%M:%S", nowtm);
     snprintf(buf, sizeof(buf), "%s.%06u", tmbuf, static_cast<uint>(tv.tv_usec));
     return std::string(buf);

code_boost/src/cxx/statistics_db.h

@@ -22,7 +22,7 @@ public:
     /*
      * Database version: Increment number on every change in the C++ code!
      */
-    static const int DB_VERSION = 12;
+    static const int DB_VERSION = 13;
 
     /*
      * Methods for writing values into database