Merge branch 'pybind_final' of stefan.schmidt/ID2T-toolkit into master

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "code_boost/src/SQLiteCpp"]
 	path = code_boost/src/SQLiteCpp
 	url = https://github.com/SRombauts/SQLiteCpp
+[submodule "code_boost/src/pybind11"]
+	path = code_boost/src/pybind11
+	url = https://github.com/pybind/pybind11

README.md

@@ -21,7 +21,7 @@ ID2T was also presented in Blackhat Europe 2017 as part of the Arsenal session (
 ## Getting Started
 
 ### Dependencies
-ID2T is written using Python 3 and C++ 11. The main logic is programmed in Python whereas performance critical components are programmed in C++11. The C++11 module uses the [Libtins](https://github.com/mfontanini/libtins/) library. The python and c++ modules interact with each other through the [Boost.Python](http://www.boost.org/doc/libs/1_62_0/libs/python/doc/html/index.html) library .
+ID2T is written using Python 3 and C++ 11. The main logic is programmed in Python whereas performance critical components are programmed in C++11. The C++11 module uses the [Libtins](https://github.com/mfontanini/libtins/) library. The python and c++ modules interact with each other through the [pybind11](https://github.com/pybind/pybind11) library.
 
 #### Required C++ Libraries/Programs
 The following packages/libraries are required to compile the ID2T C++ modules
@@ -29,10 +29,6 @@ The following packages/libraries are required to compile the ID2T C++ modules
     - ubuntu: apt install build-essential cmake
     - arch: pacman -S cmake
     - macos: brew install cmake
-* ``boost`` with the ``python`` component (minimum version 1.54)
-    - ubuntu: apt install libboost-dev libboost-python-dev
-    - arch: pacman -S boost boost-libs
-    - macos: brew install boost boost-python --with-python3
 * ``libtins`` (minimum version 3.4)
     - ubuntu: apt install libtins-dev (if you cannot find it in the official repository, install it manually from [here](https://github.com/mfontanini/libtins))
     - arch: (install from AUR, i.e. pacaur -S libtins, or manually from [here](https://github.com/mfontanini/libtins)).

code_boost/src/CMakeLists.txt

@@ -33,6 +33,9 @@ option(SQLITECPP_RUN_CPPLINT OFF)
 include_directories(SQLiteCpp/include)
 add_subdirectory(SQLiteCpp)
 
+# Include pybind11
+include_directories(pybind11/include)
+
 # Find libtins library
 FIND_LIBRARY(TINS_LIBRARY tins)
 IF(TINS_LIBRARY)
@@ -50,41 +53,14 @@ ELSE()
   MESSAGE(FATAL_ERROR "Unable to find Python libraries.")
 ENDIF()
 
-# Find and configure BOOST library
-FIND_PACKAGE(Boost 1.54 QUIET)
-IF (Boost_FOUND)
-    INCLUDE_DIRECTORIES("${Boost_INCLUDE_DIRS}")
-    MESSAGE(STATUS "Boots includes found in: " ${Boost_INCLUDE_DIRS} )
-    SET(Boost_USE_STATIC_LIBS OFF)
-    SET(Boost_USE_MULTITHREADED ON)
-    SET(Boost_USE_STATIC_RUNTIME OFF)
-    # Find the boost python 3 component
-    EXECUTE_PROCESS(COMMAND python3 --version OUTPUT_VARIABLE PY_VERSION)
-    STRING(REGEX REPLACE "Python ([0-9]+)\.([0-9]+)\.[0-9]+" "python-py\\1\\2" PY_VERSION ${PY_VERSION})
-    STRING(STRIP ${PY_VERSION} PY_VERSION)
-    SET(PYTHON_VERSIONS python3 ${PY_VERSION} python-py35 python-py34 python-py33 python-py32)
-    FOREACH(VERSION ${PYTHON_VERSIONS})
-      FIND_PACKAGE(Boost COMPONENTS ${VERSION} QUIET)
-      IF(Boost_FOUND)
-        MESSAGE(STATUS "Python Boost found as '${VERSION}'.")
-        BREAK()
-      ENDIF()
-    ENDFOREACH(VERSION)
-    IF(NOT Boost_FOUND)
-      MESSAGE(FATAL_ERROR "Python Boost component not found.")
-    ENDIF()
-ELSE ()
-    MESSAGE(FATAL_ERROR "Unable to find the Boost libraries (version 1.54 or higher).")
-ENDIF ()
-
 SET_target_properties(sqlite3 PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 ADD_LIBRARY(pcapreader SHARED ${SOURCE_FILES})
 # Libs pthread and dl are prerequisites of SQLiteCpp
-TARGET_LINK_LIBRARIES(pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl pcap)
+TARGET_LINK_LIBRARIES(pcapreader "${TINS_LIBRARY}" ${PYTHON_LIBRARIES} SQLiteCpp sqlite3 pthread dl pcap)
 
 ADD_LIBRARY(botnetcomm SHARED ${BOT_COMM_PROC_SOURCE})
-TARGET_LINK_LIBRARIES(botnetcomm ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})
+TARGET_LINK_LIBRARIES(botnetcomm ${PYTHON_LIBRARIES})
 
 IF (APPLE)
   SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
@@ -92,4 +68,4 @@ ENDIF ()
 
 # comment this out to build executable (for development)
 #ADD_EXECUTABLE(cpp-pcapreader ${SOURCE_FILES})
-#TARGET_LINK_LIBRARIES(cpp-pcapreader ${Boost_LIBRARIES} "${TINS_LIBRARY}" SQLiteCpp sqlite3 pthread dl)
+#TARGET_LINK_LIBRARIES(cpp-pcapreader "${TINS_LIBRARY}" SQLiteCpp sqlite3 pthread dl)

code_boost/src/cxx/botnet_comm_processor.cpp

@@ -1,4 +1,6 @@
 #include "botnet_comm_processor.h"
+#include <algorithm>
+#include <sstream>
 
 
 /**
@@ -25,12 +27,12 @@ botnet_comm_processor::botnet_comm_processor(){
 void botnet_comm_processor::set_messages(const py::list &messages_pyboost){
     messages.clear();
     for (int i = 0; i < len(messages_pyboost); i++){
-        py::dict msg_pyboost = py::extract<py::dict>(messages_pyboost[i]);
-        unsigned int src_id = std::stoi(py::extract<std::string>(msg_pyboost["Src"]));
-        unsigned int dst_id = std::stoi(py::extract<std::string>(msg_pyboost["Dst"]));
-        unsigned short type = (unsigned short) std::stoi(py::extract<std::string>(msg_pyboost["Type"]));
-        double time = std::stod(py::extract<std::string>(msg_pyboost["Time"]));
-        int line_no = std::stoi(py::extract<std::string>(msg_pyboost.get("LineNumber", "-1")));
+        py::dict msg_pyboost = py::cast<py::dict>(messages_pyboost[i]);
+        unsigned int src_id = std::stoi(py::cast<std::string>(msg_pyboost["Src"]));
+        unsigned int dst_id = std::stoi(py::cast<std::string>(msg_pyboost["Dst"]));
+        unsigned short type = (unsigned short) std::stoi(py::cast<std::string>(msg_pyboost["Type"]));
+        double time = std::stod(py::cast<std::string>(msg_pyboost["Time"]));
+        int line_no = std::stoi(msg_pyboost.contains("LineNumber") ? py::cast<std::string>(msg_pyboost["LineNumber"]) : "-1");
 
         abstract_msg msg = {src_id, dst_id, type, time, line_no};
         messages.push_back(std::move(msg));
@@ -95,7 +97,7 @@ unsigned int botnet_comm_processor::parse_csv(const std::string &filepath){
         cur_msg.line_no = line_no;
         // iterate over every key:value entry
         for (std::string pair; std::getline(line_stream, pair, ','); ){
-            boost::replace_all(pair, " ", "");
+            pair.erase(std::remove(pair.begin(), pair.end(), ' '), pair.end());
             std::size_t split_pos = pair.find(":");
             if (split_pos != std::string::npos){
                 std::string key = pair.substr(0, split_pos);
@@ -550,18 +552,10 @@ py::list botnet_comm_processor::convert_intervals_to_py_repr(const std::vector<c
 //     std::cout << "Src: " << message.src << "   Dst: " << message.dst << "   Type: " << message.type << "   Time: " << message.time << "   LineNumber: " << message.line_no << std::endl;
 // }
 
-
-/*
- * Comment out if executable should be build & run
- * Comment in if library should be build
- */
-
-using namespace boost::python;
-
-BOOST_PYTHON_MODULE (libbotnetcomm) {
-    class_<botnet_comm_processor>("botnet_comm_processor")
-            .def(init<list>())
-            .def(init<>())
+PYBIND11_MODULE (libbotnetcomm, m) {
+    py::class_<botnet_comm_processor>(m, "botnet_comm_processor")
+            .def(py::init<py::list>())
+            .def(py::init<>())
             .def("find_interval_from_startidx", &botnet_comm_processor::find_interval_from_startidx)
             .def("find_interval_from_endidx", &botnet_comm_processor::find_interval_from_endidx)
             .def("find_optimal_interval", &botnet_comm_processor::find_optimal_interval)

code_boost/src/cxx/botnet_comm_processor.h

@@ -7,8 +7,7 @@
 #define BOTNET_COMM_PROCESSOR_H
 
 #include <iostream>
-#include <boost/python.hpp>
-#include <boost/algorithm/string/replace.hpp>
+#include <pybind11/pybind11.h>
 #include <vector>
 #include <thread>
 #include <deque>
@@ -37,7 +36,7 @@
 /*
  * For quick usage
  */
-namespace py = boost::python;
+namespace py = pybind11;
 
 /*
  * Definition of structs
@@ -150,4 +149,4 @@ private:
 }; 
 
 
-#endif //BOTNET_COMM_PROCESSOR_H
+#endif //BOTNET_COMM_PROCESSOR_H

code_boost/src/cxx/pcap_processor.cpp

@@ -412,15 +412,15 @@ bool inline pcap_processor::file_exists(const std::string &filePath) {
  * Comment out if executable should be build & run
  * Comment in if library should be build
  */
-#include <boost/python.hpp>
+#include <pybind11/pybind11.h>
+namespace py = pybind11;
 
-using namespace boost::python;
-
-BOOST_PYTHON_MODULE (libpcapreader) {
-    class_<pcap_processor>("pcap_processor", init<std::string, std::string>())
+PYBIND11_MODULE (libpcapreader, m) {
+    py::class_<pcap_processor>(m, "pcap_processor")
+            .def(py::init<std::string, std::string>())
             .def("merge_pcaps", &pcap_processor::merge_pcaps)
             .def("collect_statistics", &pcap_processor::collect_statistics)
             .def("get_timestamp_mu_sec", &pcap_processor::get_timestamp_mu_sec)
             .def("write_to_database", &pcap_processor::write_to_database)
-            .def("get_db_version", &pcap_processor::get_db_version).staticmethod("get_db_version");
+            .def_static("get_db_version", &pcap_processor::get_db_version);
 }

code_boost/src/pybind11

@@ -0,0 +1 @@
+Subproject commit 8edc147d67ca85a93ed1f53628004528dc36a04d

resources/install_dependencies.sh

@@ -2,7 +2,7 @@
 
 install_pkg_arch()
 {
-    PACMAN_PKGS="boost boost-libs cmake python python-pip sqlite tcpdump"
+    PACMAN_PKGS="cmake python python-pip sqlite tcpdump"
 
     # Check first to avoid unnecessary sudo
     echo -e "Packages: Checking..."
@@ -43,7 +43,7 @@ install_pkg_arch()
 
 install_pkg_ubuntu()
 {
-    APT_PKGS='build-essential libboost-dev libboost-python-dev cmake python3-dev python3-pip python3-venv sqlite tcpdump libtins-dev libpcap-dev'
+    APT_PKGS='build-essential cmake python3-dev python3-pip python3-venv sqlite tcpdump libtins-dev libpcap-dev'
 
     which sudo >/dev/null
     if [ $? != 0 ]; then
@@ -67,7 +67,7 @@ install_pkg_ubuntu()
 
 install_pkg_darwin()
 {
-    BREW_PKGS="cmake python coreutils libdnet libtins sqlite boost boost-python --with-python3"
+    BREW_PKGS="cmake python coreutils libdnet libtins sqlite"
 
     # Check first to avoid unnecessary update
     echo -e "Packages: Checking..."
@@ -81,8 +81,8 @@ install_pkg_darwin()
     fi
 }
 
-# Make sure the SQLiteCpp submodule is there
-echo -e "Updating SQLiteCpp"
+# Make sure the submodules are there
+echo -e "Updating submodules"
 git submodule update --init
 
 KERNEL=$(uname)