123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650 |
- #include <iostream>
- #include <fstream>
- #include <vector>
- #include <math.h>
- #include "statistics.h"
- #include <sstream>
- #include <SQLiteCpp/SQLiteCpp.h>
- #include "statistics_db.h"
- #include "statistics.h"
- #include "utilities.h"
- using namespace Tins;
- void statistics::checkPayload(const PDU *pdu_l4) {
- if(this->getDoExtraTests()) {
-
- int pktSize = pdu_l4->size();
- int headerSize = pdu_l4->header_size();
- int payloadSize = pktSize - headerSize;
- if (payloadSize > 0)
- payloadCount++;
- }
- }
- void statistics::checkTCPChecksum(std::string ipAddressSender, std::string ipAddressReceiver, TCP tcpPkt) {
- if(this->getDoExtraTests()) {
- if(check_tcpChecksum(ipAddressSender, ipAddressReceiver, tcpPkt))
- correctTCPChecksumCount++;
- else incorrectTCPChecksumCount++;
- }
- }
- std::vector<float> statistics::calculateLastIntervalIPsEntropy(std::chrono::microseconds intervalStartTimestamp){
- if(this->getDoExtraTests()) {
- std::vector<int> IPsSrcPktsCounts;
- std::vector<int> IPsDstPktsCounts;
- std::vector<float> IPsSrcProb;
- std::vector<float> IPsDstProb;
- int pktsSent = 0, pktsReceived = 0;
- for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
- int IPsSrcPktsCount = 0;
- for (auto j = i->second.pkts_sent_timestamp.begin(); j != i->second.pkts_sent_timestamp.end(); j++) {
- if(*j >= intervalStartTimestamp)
- IPsSrcPktsCount++;
- }
- if(IPsSrcPktsCount != 0) {
- IPsSrcPktsCounts.push_back(IPsSrcPktsCount);
- pktsSent += IPsSrcPktsCount;
- }
- int IPsDstPktsCount = 0;
- for (auto j = i->second.pkts_received_timestamp.begin(); j != i->second.pkts_received_timestamp.end(); j++) {
- if(*j >= intervalStartTimestamp)
- IPsDstPktsCount++;
- }
- if(IPsDstPktsCount != 0) {
- IPsDstPktsCounts.push_back(IPsDstPktsCount);
- pktsReceived += IPsDstPktsCount;
- }
- }
- for (auto i = IPsSrcPktsCounts.begin(); i != IPsSrcPktsCounts.end(); i++) {
- IPsSrcProb.push_back((float) *i / pktsSent);
- }
- for (auto i = IPsDstPktsCounts.begin(); i != IPsDstPktsCounts.end(); i++) {
- IPsDstProb.push_back((float) *i / pktsReceived);
- }
-
- float IPsSrcEntropy = 0;
- for (unsigned i = 0; i < IPsSrcProb.size(); i++) {
- if (IPsSrcProb[i] > 0)
- IPsSrcEntropy += -IPsSrcProb[i] * log2(IPsSrcProb[i]);
- }
-
- float IPsDstEntropy = 0;
- for (unsigned i = 0; i < IPsDstProb.size(); i++) {
- if (IPsDstProb[i] > 0)
- IPsDstEntropy += -IPsDstProb[i] * log2(IPsDstProb[i]);
- }
- std::vector<float> entropies = {IPsSrcEntropy, IPsDstEntropy};
- return entropies;
- }
- else {
- return {-1, -1};
- }
- }
- std::vector<float> statistics::calculateIPsCumEntropy(){
- if(this->getDoExtraTests()) {
- std::vector <std::string> IPs;
- std::vector <float> IPsSrcProb;
- std::vector <float> IPsDstProb;
- for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
- IPs.push_back(i->first);
- IPsSrcProb.push_back((float)i->second.pkts_sent/packetCount);
- IPsDstProb.push_back((float)i->second.pkts_received/packetCount);
- }
-
- float IPsSrcEntropy = 0;
- for(unsigned i=0; i < IPsSrcProb.size();i++){
- if (IPsSrcProb[i] > 0)
- IPsSrcEntropy += - IPsSrcProb[i]*log2(IPsSrcProb[i]);
- }
-
- float IPsDstEntropy = 0;
- for(unsigned i=0; i < IPsDstProb.size();i++){
- if (IPsDstProb[i] > 0)
- IPsDstEntropy += - IPsDstProb[i]*log2(IPsDstProb[i]);
- }
- std::vector<float> entropies = {IPsSrcEntropy, IPsDstEntropy};
- return entropies;
- }
- else {
- return {-1, -1};
- }
- }
- void statistics::calculateIPIntervalPacketRate(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp){
- for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
- int IPsSrcPktsCount = 0;
- for (auto j = i->second.pkts_sent_timestamp.begin(); j != i->second.pkts_sent_timestamp.end(); j++) {
- if(*j >= intervalStartTimestamp)
- IPsSrcPktsCount++;
- }
- float interval_pkt_rate = (float) IPsSrcPktsCount * 1000000 / interval.count();
- i->second.interval_pkt_rate.push_back(interval_pkt_rate);
- if(interval_pkt_rate > i->second.max_interval_pkt_rate || i->second.max_interval_pkt_rate == 0)
- i->second.max_interval_pkt_rate = interval_pkt_rate;
- if(interval_pkt_rate < i->second.min_interval_pkt_rate || i->second.min_interval_pkt_rate == 0)
- i->second.min_interval_pkt_rate = interval_pkt_rate;
- }
- }
- void statistics::addIntervalStat(std::chrono::duration<int, std::micro> interval, std::chrono::microseconds intervalStartTimestamp, std::chrono::microseconds intervalEndTimestamp){
-
- calculateIPIntervalPacketRate(interval, intervalStartTimestamp);
-
- std::vector<float> ipEntopies = calculateLastIntervalIPsEntropy(intervalStartTimestamp);
- std::vector<float> ipCumEntopies = calculateIPsCumEntropy();
- std::string lastPktTimestamp_s = std::to_string(intervalEndTimestamp.count());
- std::string intervalStartTimestamp_s = std::to_string(intervalStartTimestamp.count());
-
- interval_statistics[lastPktTimestamp_s].pkts_count = packetCount - intervalCumPktCount;
- interval_statistics[lastPktTimestamp_s].kbytes = (float(sumPacketSize - intervalCumSumPktSize) / 1024);
- interval_statistics[lastPktTimestamp_s].payload_count = payloadCount - intervalPayloadCount;
- interval_statistics[lastPktTimestamp_s].incorrect_tcp_checksum_count = incorrectTCPChecksumCount - intervalIncorrectTCPChecksumCount;
- interval_statistics[lastPktTimestamp_s].correct_tcp_checksum_count = correctTCPChecksumCount - intervalCorrectTCPChecksumCount;
- interval_statistics[lastPktTimestamp_s].novel_ip_count = ip_statistics.size() - intervalCumNovelIPCount;
- interval_statistics[lastPktTimestamp_s].novel_ttl_count = ttl_values.size() - intervalCumNovelTTLCount;
- interval_statistics[lastPktTimestamp_s].novel_win_size_count = win_values.size() - intervalCumNovelWinSizeCount;
- interval_statistics[lastPktTimestamp_s].novel_tos_count = tos_values.size() - intervalCumNovelToSCount;
- interval_statistics[lastPktTimestamp_s].novel_mss_count = mss_values.size() - intervalCumNovelMSSCount;
- interval_statistics[lastPktTimestamp_s].novel_port_count = port_values.size() - intervalCumNovelPortCount;
- intervalPayloadCount = payloadCount;
- intervalIncorrectTCPChecksumCount = incorrectTCPChecksumCount;
- intervalCorrectTCPChecksumCount = correctTCPChecksumCount;
- intervalCumPktCount = packetCount;
- intervalCumSumPktSize = sumPacketSize;
- intervalCumNovelIPCount = ip_statistics.size();
- intervalCumNovelTTLCount = ttl_values.size();
- intervalCumNovelWinSizeCount = win_values.size();
- intervalCumNovelToSCount = tos_values.size();
- intervalCumNovelMSSCount = mss_values.size();
- intervalCumNovelPortCount = port_values.size();
- if(ipEntopies.size()>1){
- interval_statistics[lastPktTimestamp_s].ip_src_entropy = ipEntopies[0];
- interval_statistics[lastPktTimestamp_s].ip_dst_entropy = ipEntopies[1];
- }
- if(ipCumEntopies.size()>1){
- interval_statistics[lastPktTimestamp_s].ip_src_cum_entropy = ipCumEntopies[0];
- interval_statistics[lastPktTimestamp_s].ip_dst_cum_entropy = ipCumEntopies[1];
- }
- }
- void statistics::addConvStat(std::string ipAddressSender,int sport,std::string ipAddressReceiver,int dport, std::chrono::microseconds timestamp){
- conv f1 = {ipAddressReceiver, dport, ipAddressSender, sport};
- conv f2 = {ipAddressSender, sport, ipAddressReceiver, dport};
-
- if (conv_statistics.count(f1)>0){
- conv_statistics[f1].pkts_count++;
- if(conv_statistics[f1].pkts_count<=3)
- conv_statistics[f1].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics[f1].pkts_timestamp.back()));
- conv_statistics[f1].pkts_timestamp.push_back(timestamp);
- }
-
- else{
- conv_statistics[f2].pkts_count++;
- if(conv_statistics[f2].pkts_timestamp.size()>0 && conv_statistics[f2].pkts_count<=3 )
- conv_statistics[f2].interarrival_time.push_back(std::chrono::duration_cast<std::chrono::microseconds> (timestamp - conv_statistics[f2].pkts_timestamp.back()));
- conv_statistics[f2].pkts_timestamp.push_back(timestamp);
- }
- }
- void statistics::incrementMSScount(std::string ipAddress, int mssValue) {
- mss_values[mssValue]++;
- mss_distribution[{ipAddress, mssValue}]++;
- }
- void statistics::incrementWinCount(std::string ipAddress, int winSize) {
- win_values[winSize]++;
- win_distribution[{ipAddress, winSize}]++;
- }
- void statistics::incrementTTLcount(std::string ipAddress, int ttlValue) {
- ttl_values[ttlValue]++;
- ttl_distribution[{ipAddress, ttlValue}]++;
- }
- void statistics::incrementToScount(std::string ipAddress, int tosValue) {
- tos_values[tosValue]++;
- tos_distribution[{ipAddress, tosValue}]++;
- }
- void statistics::incrementProtocolCount(std::string ipAddress, std::string protocol) {
- protocol_distribution[{ipAddress, protocol}]++;
- }
- int statistics::getProtocolCount(std::string ipAddress, std::string protocol) {
- return protocol_distribution[{ipAddress, protocol}];
- }
- void statistics::incrementPortCount(std::string ipAddressSender, int outgoingPort, std::string ipAddressReceiver,
- int incomingPort) {
- port_values[outgoingPort]++;
- port_values[incomingPort]++;
- ip_ports[{ipAddressSender, "out", outgoingPort}]++;
- ip_ports[{ipAddressReceiver, "in", incomingPort}]++;
- }
- statistics::statistics(void) {
- }
- void statistics::assignMacAddress(std::string ipAddress, std::string macAddress) {
- ip_mac_mapping[ipAddress] = macAddress;
- }
- void statistics::addIpStat_packetSent(std::string filePath, std::string ipAddressSender, std::string ipAddressReceiver, long bytesSent, std::chrono::microseconds timestamp) {
-
- if(ip_statistics[ipAddressSender].pkts_sent==0){
-
- ip_statistics[ipAddressSender].ip_class = getIPv4Class(ipAddressSender);
- }
-
-
- if(ip_statistics[ipAddressReceiver].pkts_received==0){
-
- ip_statistics[ipAddressReceiver].ip_class = getIPv4Class(ipAddressReceiver);
- }
-
- ip_statistics[ipAddressSender].kbytes_sent += (float(bytesSent) / 1024);
- ip_statistics[ipAddressSender].pkts_sent++;
- ip_statistics[ipAddressSender].pkts_sent_timestamp.push_back(timestamp);
-
-
- ip_statistics[ipAddressReceiver].kbytes_received += (float(bytesSent) / 1024);
- ip_statistics[ipAddressReceiver].pkts_received++;
- ip_statistics[ipAddressReceiver].pkts_received_timestamp.push_back(timestamp);
- }
- void statistics::setTimestampFirstPacket(Tins::Timestamp ts) {
- timestamp_firstPacket = ts;
- }
- void statistics::setTimestampLastPacket(Tins::Timestamp ts) {
- timestamp_lastPacket = ts;
- }
- Tins::Timestamp statistics::getTimestampFirstPacket() {
- return timestamp_firstPacket;
- }
- Tins::Timestamp statistics::getTimestampLastPacket() {
- return timestamp_lastPacket;
- }
- int statistics::getPacketCount() {
- return packetCount;
- }
- int statistics::getSumPacketSize() {
- return sumPacketSize;
- }
- float statistics::getAvgPacketSize() const {
-
- return (sumPacketSize / packetCount) / 1024;
- }
- void statistics::addPacketSize(uint32_t packetSize) {
- sumPacketSize += ((float) packetSize);
- }
- void statistics::setDoExtraTests(bool var) {
- doExtraTests = var;
- }
- bool statistics::getDoExtraTests() {
- return doExtraTests;
- }
- std::string statistics::getCaptureDurationTimestamp() const {
-
- time_t t = (timestamp_lastPacket.seconds() - timestamp_firstPacket.seconds());
- time_t ms = (timestamp_lastPacket.microseconds() - timestamp_firstPacket.microseconds());
- long int hour = t / 3600;
- long int remainder = (t - hour * 3600);
- long int minute = remainder / 60;
- long int second = (remainder - minute * 60) % 60;
- long int microseconds = ms;
-
- char out[64];
- sprintf(out, "%02ld:%02ld:%02ld.%06ld ", hour, minute, second, microseconds);
- return std::string(out);
- }
- float statistics::getCaptureDurationSeconds() const {
- timeval d;
- d.tv_sec = timestamp_lastPacket.seconds() - timestamp_firstPacket.seconds();
- d.tv_usec = timestamp_lastPacket.microseconds() - timestamp_firstPacket.microseconds();
- char tmbuf[64], buf[64];
- auto nowtm = localtime(&(d.tv_sec));
- strftime(tmbuf, sizeof(tmbuf), "%S", nowtm);
- snprintf(buf, sizeof(buf), "%s.%06u", tmbuf, (uint) d.tv_usec);
- return std::stof(std::string(buf));
- }
- std::string statistics::getFormattedTimestamp(time_t seconds, suseconds_t microseconds) const {
- timeval tv;
- tv.tv_sec = seconds;
- tv.tv_usec = microseconds;
- char tmbuf[64], buf[64];
- auto nowtm = localtime(&(tv.tv_sec));
- strftime(tmbuf, sizeof(tmbuf), "%Y-%m-%d %H:%M:%S", nowtm);
- snprintf(buf, sizeof(buf), "%s.%06u", tmbuf, (uint) tv.tv_usec);
- return std::string(buf);
- }
- ip_stats statistics::getStatsForIP(std::string ipAddress) {
- float duration = getCaptureDurationSeconds();
- entry_ipStat ipStatEntry = ip_statistics[ipAddress];
- ip_stats s;
- s.bandwidthKBitsIn = (ipStatEntry.kbytes_received / duration) * 8;
- s.bandwidthKBitsOut = (ipStatEntry.kbytes_sent / duration) * 8;
- s.packetPerSecondIn = (ipStatEntry.pkts_received / duration);
- s.packetPerSecondOut = (ipStatEntry.pkts_sent / duration);
- s.AvgPacketSizeSent = (ipStatEntry.kbytes_sent / ipStatEntry.pkts_sent);
- s.AvgPacketSizeRecv = (ipStatEntry.kbytes_received / ipStatEntry.pkts_received);
- return s;
- }
- void statistics::incrementPacketCount() {
- packetCount++;
- }
- void statistics::printStats(std::string ipAddress) {
- std::stringstream ss;
- ss << std::endl;
- ss << "Capture duration: " << getCaptureDurationSeconds() << " seconds" << std::endl;
- ss << "Capture duration (HH:MM:SS.mmmmmm): " << getCaptureDurationTimestamp() << std::endl;
- ss << "#Packets: " << packetCount << std::endl;
- ss << std::endl;
-
- if (ipAddress != "") {
- entry_ipStat e = ip_statistics[ipAddress];
- ss << "\n----- STATS FOR IP ADDRESS [" << ipAddress << "] -------" << std::endl;
- ss << std::endl << "KBytes sent: " << e.kbytes_sent << std::endl;
- ss << "KBytes received: " << e.kbytes_received << std::endl;
- ss << "Packets sent: " << e.pkts_sent << std::endl;
- ss << "Packets received: " << e.pkts_received << "\n\n";
- ip_stats is = getStatsForIP(ipAddress);
- ss << "Bandwidth IN: " << is.bandwidthKBitsIn << " kbit/s" << std::endl;
- ss << "Bandwidth OUT: " << is.bandwidthKBitsOut << " kbit/s" << std::endl;
- ss << "Packets per second IN: " << is.packetPerSecondIn << std::endl;
- ss << "Packets per second OUT: " << is.packetPerSecondOut << std::endl;
- ss << "Avg Packet Size Sent: " << is.AvgPacketSizeSent << " kbytes" << std::endl;
- ss << "Avg Packet Size Received: " << is.AvgPacketSizeRecv << " kbytes" << std::endl;
- }
- std::cout << ss.str();
- }
- void statistics::writeToDatabase(std::string database_path) {
-
- float duration = getCaptureDurationSeconds();
- long sumPacketsSent = 0, senderCountIP = 0;
- float sumBandwidthIn = 0.0, sumBandwidthOut = 0.0;
- for (auto i = ip_statistics.begin(); i != ip_statistics.end(); i++) {
- sumPacketsSent += i->second.pkts_sent;
-
- sumBandwidthIn += (i->second.kbytes_received / duration);
- sumBandwidthOut += (i->second.kbytes_sent / duration);
- senderCountIP++;
- }
- float avgPacketRate = (packetCount / duration);
- long avgPacketSize = getAvgPacketSize();
- if(senderCountIP>0) {
- long avgPacketsSentPerHost = (sumPacketsSent / senderCountIP);
- float avgBandwidthInKBits = (sumBandwidthIn / senderCountIP) * 8;
- float avgBandwidthOutInKBits = (sumBandwidthOut / senderCountIP) * 8;
-
- statistics_db db(database_path);
- db.writeStatisticsFile(packetCount, getCaptureDurationSeconds(),
- getFormattedTimestamp(timestamp_firstPacket.seconds(), timestamp_firstPacket.microseconds()),
- getFormattedTimestamp(timestamp_lastPacket.seconds(), timestamp_lastPacket.microseconds()),
- avgPacketRate, avgPacketSize, avgPacketsSentPerHost, avgBandwidthInKBits,
- avgBandwidthOutInKBits);
- db.writeStatisticsIP(ip_statistics);
- db.writeStatisticsTTL(ttl_distribution);
- db.writeStatisticsIpMac(ip_mac_mapping);
- db.writeStatisticsPorts(ip_ports);
- db.writeStatisticsProtocols(protocol_distribution);
- db.writeStatisticsMSS(mss_distribution);
- db.writeStatisticsToS(tos_distribution);
- db.writeStatisticsWin(win_distribution);
- db.writeStatisticsConv(conv_statistics);
- db.writeStatisticsInterval(interval_statistics);
- }
- else {
-
- std::cout<<"ERROR: Statistics could not be collected from the input PCAP!"<<"\n";
- return;
- }
- }
|