integrated most of code, TODO: test and fix

M-star.py

@@ -0,0 +1,45 @@
+import logging
+import sys
+from vendors.debian.DebianModel import DebianModel
+
+from pymongo import MongoClient
+
+##### GLOBAL VARIABLES #####
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+# Increase the recursion limit by much to allow bs to parse large files ()
+sys.setrecursionlimit(6000)
+
+secondsperday = 86400
+verbosity = 1
+
+###### FUNCTIONS ######
+
+
+def aptsec_help():
+    """
+    :return:
+    """
+    print('See manual for correct usage!')
+
+
+'''
+TODO:
+    - add parameterized extensions
+        - for input config_default.txt file
+        - for action
+        - vendors ( OPTIONAL )
+    
+'''
+
+
+def __main__(configfile='config_default.txt', vendorname='debian', action='help'):
+    # support only debian
+    if vendorname is 'debian':
+        model = DebianModel(action)
+    else:
+        print("Only debian vendors are supported for now.")
+        sys.exit(1)
+
+
+__main__(action='update')

app-sec.py

@@ -1,111 +0,0 @@
-import sys
-import os
-import logging
-import configparser
-import datetime
-from pymongo import MongoClient
-import json
-
-##### GLOBAL VARIABLES #####
-logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
-
-# Increase the recursion limit by much to allow bs to parse large files ()
-sys.setrecursionlimit(6000)
-
-# Load config file as library
-config = configparser.ConfigParser()
-
-secondsperday = 86400
-now = datetime.datetime.now()
-verbosity = 1
-
-###### FUNCTIONS ######
-
-
-def load_state():
-    """
-    Load state, different from DBs in that we always need it.
-    Retrieves the cached state for current configuration.
-    :return:  state , error number
-    """
-    cache = config['DIR']['cache_dir'] + 'state'
-    err = 0
-    state = dict()
-
-    try:
-        with open(cache) as json_data:
-            state = json.load(json_data)
-    except FileNotFoundError:
-        # Load default state - start from the beginning
-        print('File not found in: '+config['DIR']['cache_dir'] + 'state')
-        print('Loading default state.')
-        state['cache_dir'] = cache
-        state['next_adv'] = 0
-        state['next_fsa'] = 0
-        state['Packages'] = ''
-        state['Sources'] = ''
-        state['Sha1Sums'] = ''
-        err += 1
-
-    return state, err
-
-
-def load_DBs():
-    cache = config['DIR']['cache_dir']
-
-    tables = ['dsatable', 'src2dsa', 'dsa2cve', 'cvetable', 'src2deps', 'src2month', 'src2sloccount', 'src2pop']
-    result = []
-
-    for i in range(0, len(tables)):
-        try:
-            with open(cache + tables[i]) as t:
-                result.append(json.load(t))
-        except (IOError, ValueError):
-            print('read cache '+tables[i]+' failed!! Maybe first run of the system?')
-            result.append("zero")
-
-    return tuple(result)
-
-
-def aptsec_help():
-    """
-    :return:
-    """
-    print('See manual for correct usage!')
-
-
-'''
-TODO:
-    - add parameterized extensions
-        - for input config file
-        - for action
-        - vendor ( OPTIONAL )
-    
-'''
-
-
-def __main__(configfile='config', vendorname='debian', action='help'):
-    # initialize structures for further computation
-    if not config.read(configfile):
-        raise IOError('Cannot open configuration file')
-    client = MongoClient()
-    dsatable = dict()
-    cve_db = client.cvedb
-
-    (state, err) = load_state()
-    state['vendor'] = vendorname
-
-    if action == 'update':
-        (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = load_DBs()
-    elif action == 'status':
-        (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = load_DBs()
-        #aptsec_status(sys.argv[2])
-    elif action == 'show':
-        (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = load_DBs()
-        #src2sum = plot_all(src2month)
-        #save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, src2sum)
-    else:
-        aptsec_help()
-
-
-__main__(action='update')

vendors/debian/CVEParse.py

@@ -0,0 +1,67 @@
+import logging, sys
+import re
+import datetime
+
+
+class CVEParse:
+    """
+    Functions for downloading and parsing Common Vulnerability DB data.
+    """
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+    @staticmethod
+    def correctCVE(cve_id):
+        cve_id_new = cve_id
+        if cve_id == 2116:
+            cve_id_new = 1921
+
+        return cve_id_new
+
+    @staticmethod
+    def fetchCVE(cve_id, client):
+        """
+        Get details of given CVE entry from NIST DB - we use cve-search and mongodb
+        """
+        logging.info('Fetching CVE: ' + cve_id + '\n')
+
+        # Make this subtitution for some reason
+        cve_id = re.sub('^CAN', 'CVE', cve_id)
+        cve_id = CVEParse.correctCVE(cve_id)
+        ##
+        ## get CVE Scores from db
+        ##
+        db = client.cvedb
+        collection = db.cves
+        cve = collection.find_one({"id": cve_id})
+
+        if cve == '':
+            logging.warning('CVE not found in mongodb')
+        return cve
+
+
+        # Check for error pages: referenced but unpublished CVEs :-/
+
+
+    @staticmethod
+    def parseCVE(cve_id, cve):
+        """
+        Get CVE severity rating and report date, and return
+        (date base-score impact-score exploit-score)
+        """
+        # use worst case info as defaults
+        cve_date = datetime.datetime.now()
+        cve_base = 10
+        cve_impact = 10
+        cve_exploit = 10
+
+        try:
+            if cve == None:
+                print('CVE' + str(cve_id) + ' not yet reported, getting default values')
+                return (cve_date, cve_base, cve_impact, cve_exploit)
+            else:
+                cve_date = cve['Published']
+                cve_base = cve['cvss']
+        except KeyError:
+            print('CVE ' + cve_id + ' not parsed correctly')
+
+        return (cve_date, cve_base, cve_impact, cve_exploit)

vendors/debian/DebianAdvisory.py

@@ -0,0 +1,265 @@
+import logging
+import datetime
+import os
+import re
+import urllib.request
+from bs4 import BeautifulSoup
+from bs4 import NavigableString
+
+
+class DebianAdvisory:
+
+    @staticmethod
+    def checkDSAs(state, config):
+        """Try to find new DSAs by iteration, return table of DSAs to process"""
+        dsatable = dict()
+        next_dsa = int(state['next_adv'])
+        # state implemented as dictionary
+        base_url = config['URL']['dsa_base_url']
+
+        logging.info('Checking for new DSAs.. \n')
+
+        if next_dsa < int(config['DSA']['first_dsa']):
+            logging.debug('Cache was deleted, starting at DSA ' + str(next_dsa) + '\n')
+            next_dsa = int(config['DSA']['first_dsa'])
+
+        next_dsa2string = '%03d' % next_dsa
+
+        blacklist = map(str.strip, config['DSA']['blacklist'].split(','))
+        if DebianAdvisory.blacklistedDSA('DSA-' + next_dsa2string, blacklist):
+            next_dsa += 1
+
+        dsa = DebianAdvisory.fetchDSA(next_dsa, base_url)
+
+
+        while dsa != '':
+            logging.debug('Got DSA-' + str(next_dsa) + '\n')
+            soup = BeautifulSoup(dsa, 'html.parser')
+            # crop the DSA from unecessary weight
+            dsa = soup.find(id="content")
+            if dsa == '':
+                raise NameError('html file format unexpected')
+            dsatable[next_dsa] = str(dsa)
+            next_dsa += 1
+            if DebianAdvisory.blacklistedDSA('DSA-' + str(next_dsa), blacklist):
+                next_dsa += 1
+            dsa = DebianAdvisory.fetchDSA(next_dsa, base_url)
+
+        state['next_adv'] = next_dsa
+        return dsatable
+
+    @staticmethod
+    def blacklistedDSA(dsa_id, blacklist):
+        """Should this advisory be skipped?"""
+        if dsa_id in blacklist:
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def fetchDSA(dsa_id, base_url):
+        """Fetches a given dsa from the url."""
+        year = 2000
+        now = datetime.datetime.now()
+        current_year = now.year
+
+        logging.info('Fetching DSA-%d records\n', dsa_id)
+
+        if dsa_id >= 3751:
+            year = 2017
+        elif dsa_id >= 3431:
+            year = 2016
+        elif dsa_id >= 3118:
+            year = 2015
+        elif dsa_id >= 2832:
+            year = 2014
+        elif dsa_id >= 2597:
+            year = 2013
+        elif dsa_id >= 2377:
+            year = 2012
+        elif dsa_id >= 2140:
+            year = 2011
+        elif dsa_id >= 1965:
+            year = 2010
+        elif dsa_id >= 1694:
+            year = 2009
+        elif dsa_id >= 1443:
+            year = 2008
+        elif dsa_id >= 1245:
+            year = 2007
+        elif dsa_id >= 929:
+            year = 2006
+        elif dsa_id >= 622:
+            year = 2005
+        elif dsa_id >= 406:
+            year = 2004
+        elif dsa_id >= 220:
+            year = 2003
+        elif dsa_id >= 96:
+            year = 2002
+        elif dsa_id >= 11:
+            year = 2001
+
+        dsa_id2string = '%03d' % dsa_id
+
+        print(dsa_id2string)
+        flag = True
+        while flag:
+            try:
+                flag = False
+                logging.info('Opening url: ' + base_url + str(year) + '/dsa-' + dsa_id2string + '\n')
+                req = urllib.request.urlopen(base_url + str(year) + '/dsa-' + dsa_id2string)
+                charset = req.info().get_content_charset()
+                if charset is None:
+                    charset = 'utf-8'
+                dsa = req.read().decode(charset)
+                return dsa
+            except urllib.error.HTTPError as err:
+                if year < current_year:
+                    year += 1
+                    flag = True
+                else:
+                    dsa = ''
+                    return dsa
+
+    @staticmethod
+    def parseDSAhtml(dsa):
+        dsa_names = []
+        dsa_CVEs = []
+        # Date Reported -> dsa_date
+        soup = BeautifulSoup(dsa, 'html.parser')
+        tmp = soup.find("dt", string=re.compile(".*Date Repo.*:"))
+        tmp = str(tmp.find_next().contents[0])
+        # dsa_date = tmp.split()
+        # date in datetime python format
+        dsa_date = datetime.datetime.strptime(tmp, "%d %b %Y")
+        if not dsa_date:
+            print('Unable to extract date. Raising exception...')
+            raise NameError('DSA parsing problem!')
+
+        # Affected Packages -> dsa_names
+        # print(dsa)
+        tmp = soup.find("dt", string=re.compile("Affected Packages:"))
+        tmp = tmp.find_next().contents
+        # Need to check with multiple vulnerable packages
+        for i in tmp:
+            if (not isinstance(i, NavigableString)) and i.has_attr('href'):
+                # greedy 'and' operation assumed
+                unified = DebianAdvisory.unifySrcName(i.string)
+                dsa_names.append(unified)
+                pass
+        if not dsa_names:
+            print('Unable to find src package in DSA. unnamed package...')
+            dsa_names.append('unnamed')
+            print('Unnamed dsa:' + str(dsa) + '\n')
+
+        # Security database references (CVEs) -> dsa_CVEs
+        tmp = soup.find("dt", string=re.compile("Security database references:"))
+        tmp = tmp.find_next().descendants
+        for i in tmp:
+            if (not isinstance(i, NavigableString)) and i.has_attr('href'):
+                # don't count bug database
+                if not re.compile("^Bug*").match(i.string):
+                    dsa_CVEs.append(i.string)
+
+        return dsa_names, dsa_date, dsa_CVEs
+
+    @staticmethod
+    def unifySrcName(name):
+        """Track renamed packages here, easy but manual. We should look into ways to automate this
+        TODO: it should map to the most recent version, not unversioned
+        TODO: we can partially automate this..
+        make all lower-case
+        replace -X.Y version numbers by highest encounter(?)
+        handle special cases like xfree86
+        """
+        lowername = name.lower()
+        suf = os.path.join(os.path.dirname(__file__), 'src_name_unifications.txt')
+        with open(suf) as su:
+            for line in su:
+                sp_line = line.strip().split("->")
+                print(sp_line)
+                if re.compile(sp_line[0]).match(name):
+                    return sp_line[1]
+        return name
+
+
+    def fixDSAquirks(dsa_id, dsa_state):
+        """
+        TODO:
+        Static map to correct errors in DSAs
+        Return fixed list of CVE IDs or 0 to skip DSA
+        This code is still experimental
+        """
+        new_names = dsa_state[0]
+        new_date = dsa_state[1]
+        new_cves = dsa_state[2]
+
+        if dsa_id == 85:
+            new_cves = ["CVE-2001-1562", "LOCAL-03/04/05", "LOCAL-08/24/08"]
+        elif dsa_id == 745:
+            newcves = ["CVE-2005-1921", "CVE-2005-2106", "CVE-2005-1921"]
+        elif dsa_id == 1095:
+            new_cves = ["CVE-2006-0747", "CVE-2006-1861", "CVE-2006-2661"]
+        elif dsa_id == 1284:
+            new_cves = ["CVE-2007-1320", "CVE-2007-1321", "CVE-2007-1322", "CVE-2007-2893", "CVE-2007-1366"]
+        elif dsa_id == 1502:
+            new_cves = ["CVE-2007-2821", "CVE-2007-3238", "CVE-2008-0193", "CVE-2008-0194"]
+        elif dsa_id == 1706:
+            new_cves = ["CVE-2009-0135", "CVE-2009-0136"]
+        elif dsa_id == 1757:
+            new_cves = ["CVE-2007-2383", "CVE-2008-7220", "CVE-2009-1208"]
+        elif dsa_id == 1896:
+            new_cves = ["CVE-2009-3474", "CVE-2009-3475", "CVE-2009-3476"]
+        elif dsa_id == 1931:
+            new_cves = ["CVE-2009-0689", "CVE-2009-2463"]
+        elif dsa_id == 1989:
+            new_cves = ["CVE-2010-0789"]
+        elif dsa_id == 1941:
+            new_cves = ["CVE-2009-0755", "CVE-2009-3903", "CVE-2009-3904", "CVE-2009-3905", "CVE-2009-3606",
+                        "CVE-2009-3607", "CVE-2009-3608", "CVE-2009-3909", "CVE-2009-3938"]
+        elif dsa_id == 2004:
+            new_cves = ["CVE-2010-0787", "CVE-2010-0547"]
+        elif dsa_id == 2008:
+            new_cves = ["LOCAL-02/23/10", "LOCAL-02/23/10", "LOCAL-02/23/10", "LOCAL-02/23/10"]
+        elif dsa_id == 2043:
+            new_cves = ["CVE-2010-2062"]
+        elif dsa_id == 2044:
+            new_cves = ["CVE-2010-2062"]
+        elif dsa_id == 2056:
+            new_cves = ["CVE-2010-2155", "CVE-2009-4882"]
+        elif dsa_id == 2092:
+            new_cves = ["CVE-2010-1625", "CVE-2010-1448", "CVE-2009-4497"]
+        elif dsa_id == 2098:
+            new_cves = ["CVE-2010-3659", "CVE-2010-3660", "CVE-2010-3661", "CVE-2010-3662", "CVE-2010-3663",
+                        "CVE-2010-3664", "CVE-2010-3665", "CVE-2010-3666", "CVE-2010-3667", "CVE-2010-3668",
+                        "CVE-2010-3669", "CVE-2010-3670", "CVE-2010-3671", "CVE-2010-3672", "CVE-2010-3673",
+                        "CVE-2010-3674"]
+        elif dsa_id == 2103:
+            new_cves = ["CVE-2010-3076"]
+        elif dsa_id == 2218:
+            new_cves = ["CVE-2011-1684"]
+        elif dsa_id == 2229:
+            new_cves = ["CVE-2005-4494", "CVE-2006-0517", "CVE-2006-0518", "CVE-2006-0519", "CVE-2006-0625",
+                        "CVE-2006-0626", "CVE-2006-1295", "CVE-2006-1702", "CVE-2007-4525", "CVE-2008-5812",
+                        "CVE-2008-5813", "CVE-2009-3041"]
+        elif dsa_id == 2261:
+            new_cves = ["CVE-2009-4078", "CVE-2009-4079", "CVE-2009-4059", "LOCAL-12/30/10", "LOCAL-12/30/10"]
+        elif dsa_id == 2262:
+            new_cves = ["LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-05/18/11",
+                        "LOCAL-05/18/11"]
+        elif dsa_id == 2286:
+            new_names = ["phpmyadmin"]
+            print(str(dsa_id) + 'whatsapp??')
+        elif dsa_id == 1977:
+            new_names = ["python3.5"]
+        elif (
+                                    dsa_id == 47 or dsa_id == 479 or dsa_id == 480 or dsa_id == 482 or dsa_id == 489 or dsa_id == 491 or dsa_id == 495):
+            new_names = ["linux"]
+            print('Substitution successful')
+        elif dsa_id == 2289:
+            new_cves = ["LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11",
+                        "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11",
+                        "LOCAL-07/27/11", "LOCAL-07/27/11"]
+
+        return new_names, new_date, new_cves

vendors/debian/DebianModel.py

@@ -0,0 +1,360 @@
+import configparser
+import json
+import datetime
+import logging
+from pymongo import MongoClient
+import numpy as np
+import os
+from dateutil import parser
+
+from .DebianAdvisory import DebianAdvisory
+from .CVEParse import CVEParse
+
+class DebianModel:
+    """
+    This class represents M-Star debian module. It is responsible for handling debian package infos.
+    """
+    module_path = os.path.dirname(__file__)
+
+    def __init__(self, action, configfile=os.path.join(module_path, 'config_default.txt')):
+        ## DBs to track
+        self.dsatable = dict()
+        self.src2dsa = dict()
+        self.dsa2cve = dict()
+        self.cvetable = dict()
+        self.src2month = dict()
+        self.src2sloccount = dict()
+        self.src2pop = dict()
+        self.src2deps = dict()
+        self.pkg_with_cvss = dict()
+        self.src2sum = dict()
+
+        ## config
+        self.configfile = configfile
+        self.config = configparser.ConfigParser()
+        if not self.config.read(configfile):
+            raise IOError('Cannot open configuration file: ')
+        (self.state, self.err) = self.load_state()
+
+        client = MongoClient()
+
+        if action == 'update':
+            (self.dsatable, self.src2dsa, self.dsa2cve, self.cvetable, self.src2month, self.src2sloccount, self.src2pop, self.src2deps) = self.load_dbs()
+            self.update_dbs(self.dsatable, client, self.src2dsa, self.dsa2cve, self.src2month, self.cvetable, self.pkg_with_cvss)
+            self.save_DBs(self.dsatable, self.src2dsa, self.dsa2cve, self.cvetable, self.src2month, self.src2sloccount, self.src2pop, self.src2deps, self.src2sum)
+            self.save_state(self.state)
+            # lstm.predict(src2month, src2sloccount, src2pop, src2deps)
+        elif action == 'status':
+            (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = self.load_dbs()
+            # aptsec_status(sys.argv[2])
+        elif action == 'show':
+            (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = self.load_dbs()
+            # src2sum = plot_all(src2month)
+            # save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, src2sum)
+        else:
+            self.print_help(self)
+
+    def load_dbs(self):
+        """
+        Loads the required databases into the model. Can either be implemented as read from file, or read from DB.
+        Currently reading it from files in the cache folder.
+        """
+        cache_dir = os.path.join( self.module_path, self.config['DIR']['cache_dir'])
+
+        tables = ['dsatable', 'src2dsa', 'dsa2cve', 'cvetable', 'src2deps', 'src2month', 'src2sloccount', 'src2pop']
+        result = []
+
+        for i in range(0, len(tables)):
+            try:
+                with open(os.path.join(cache_dir, tables[i])) as t:
+                    result.append(json.load(t))
+            except (IOError, ValueError):
+                print('Read cache ' + tables[i] + ' failed!! Maybe first run of the system?')
+                result.append(dict())
+
+        return tuple(result)
+
+    def save_DBs(self, dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, src2sum):
+        cache_dir = os.path.join(self.module_path, self.config['DIR']['cache_dir'])
+
+        tables = ['dsatable', 'src2dsa', 'dsa2cve', 'cvetable', 'src2deps', 'src2sloccount', 'src2pop']
+
+        for i in range(0, len(tables)):
+            try:
+                with open(os.path.join(cache_dir, tables[i]), 'w') as t:
+                    json.dump(dsatable, t, default=self.converter)
+            except IOError:
+                print('write cache dsatable failed!! Fatal error')
+
+        cache_src2month = os.path.join(cache_dir, 'src2month')
+        int_list = dict()
+
+        for element in src2month:
+            for i in range(len(src2month[element])):
+                if element in int_list:
+                    int_list[element].append(int(src2month[element][i]))
+                else:
+                    int_list[element] = []
+                    int_list[element].append(int(src2month[element][i]))
+        try:
+            with open(cache_src2month, 'w') as fp:
+                json.dump(int_list, fp, default=self.converter)
+        except IOError:
+            print('write cache src2month failed!! Fatal error')
+
+    def save_state(self, state):
+        """Save state, different from DBs in that we always need it"""
+        state_file = os.path.join(self.module_path, self.config['DIR']['cache_dir'], 'state')
+
+        try:
+            with open(state_file, 'w') as sf:
+                json.dump(state, sf)
+        except IOError:
+            print('Write cache state failed!! Fatal error')
+
+    def converter(self, o):
+        """Help for save_DBs"""
+        if isinstance(o, datetime.datetime) or isinstance(o, datetime.timedelta):
+            return str(o)
+        if isinstance(o, np.float):
+            return o.astype(int)
+
+    def update_dbs(self, dsatable, client, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss):
+        now = datetime.datetime.now()
+        new_adv = DebianAdvisory.checkDSAs(self.state, self.config)
+
+        for id in new_adv:
+            if id in dsatable:
+                logging.info(self.state['vendor'] + ' advisory ' + id + ' already known.\n')
+            else:
+                ## store advisory and parse it
+                dsatable[id] = new_adv[id]
+                self.updateCVETables(id, dsatable, self.state, src2dsa, dsa2cve, cvetable, client)
+
+        # recompute all pkg statistics
+        for srcpkg in src2dsa:
+            self.processCVEs(srcpkg, now, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss, self.config)
+        
+        return 0
+
+    def updateCVETables(self, myid, dsatable, state, src2dsa, dsa2cve, cvetable, client):
+
+        logging.info('Updating vulnerability database with advisory ' + state['vendor'] + str(myid) + ' \n')
+
+        adv = dsatable[myid]
+        dsastats = DebianAdvisory.parseDSAhtml(adv)
+
+        dsastats = DebianAdvisory.fixDSAquirks(myid, dsastats)
+
+        for srcpkg in dsastats[0]:
+            if srcpkg in src2dsa:
+                src2dsa[srcpkg].append(myid)
+            else:
+                src2dsa[srcpkg] = []
+                src2dsa[srcpkg].append(myid)
+
+            dsa2cve[str(myid)] = dsastats[2]
+
+
+        for cve_id in dsastats[2]:
+            # No fetch CVE We use mongodb and cve-search
+            cve = CVEParse.fetchCVE(cve_id, client)
+            cvestats = CVEParse.parseCVE(cve_id, cve)
+            finaldate = cvestats[0]
+
+            if cvestats[0] > dsastats[1] or cvestats[0] == 0:
+                finaldate = dsastats[1]
+
+            cvedata = (finaldate, dsastats[1] - finaldate, cvestats[1], cvestats[2], cvestats[3])
+            cvetable[cve_id] = cvedata
+
+        return cvetable
+
+    @staticmethod
+    def print_help():
+        """
+        Prints help message to this vendor model.
+        """
+        print("Debian mstar model supports only update status and show actions.")
+
+    def load_state(self):
+        """
+        Load state, different from DBs in that we always need it.
+        Retrieves the cached state for current configuration.
+        :return:  state , error number
+        """
+        state_file = os.path.join(self.module_path, self.config['DIR']['cache_dir'], 'state')
+        err = 0
+
+        try:
+            with open(state_file) as json_data:
+                state = json.load(json_data)
+        except FileNotFoundError:
+            # Load default state - start from the beginning
+            print('File not found in: ' + state_file)
+            print('Loading default state.')
+            state = dict()
+            state['cache_dir'] = 'cache/'
+            state['next_adv'] = 0
+            state['next_fsa'] = 0
+            state['Packages'] = ''
+            state['Sources'] = ''
+            state['Sha1Sums'] = ''
+            err += 1
+
+        return state, err
+
+
+    def processCVEs(self, pkg, now, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss, config):
+        """
+        compute and store MTBF, MTBR and Scores of each src pkg
+        output: %src2mtbf
+        (srcpkg=> ())
+        """
+        stats = [now, 0, 0, 0, 0, 0, 0]
+        mylambda = config['TRUST']['lambda']
+        cvestats = dict()
+        logging.info('Processing package: ' + pkg + '.\n')
+
+        ## keep track of the number of low-medium-high severity vulnerabilities
+        ## TODO see how cvss affects vulnerability prediction - if some packages show patterns
+        temp_cvss = 0.0
+        with_cvss = dict()
+
+        ## To eliminate duplicate cves
+        haveseen = dict()
+
+        ## cvestats = (date: number)
+        for dsa_id in src2dsa[pkg]:
+            for cve_id in dsa2cve[str(dsa_id)]:
+                if cve_id in haveseen:
+                    continue
+                else:
+                    haveseen[cve_id] = 1
+                    tt = cvetable[cve_id][0]
+                    if tt in cvestats:
+                        cvestats[tt] += 1
+                    else:
+                        cvestats[tt] = 1
+            stats[1] += 1
+
+        ## Date at the moment taken from CVE? - not sure.
+
+        ## with_cvss = (date: number low, number med, number high)
+        for dsa_id in src2dsa[pkg]:
+            for cve_id in dsa2cve[str(dsa_id)]:
+                tt = cvetable[cve_id][0]
+                try:
+                    temp_cvss = float(cvetable[cve_id][2])
+                except TypeError:
+                    print(cve_id)
+                    continue
+
+                if tt in with_cvss:
+                    if (temp_cvss < 4.0):
+                        with_cvss[tt][0] += 1
+                    elif (temp_cvss < 7.0):
+                        with_cvss[tt][1] += 1
+                    else:
+                        with_cvss[tt][2] += 1
+                else:
+                    with_cvss[tt] = [0, 0, 0]
+                    if (temp_cvss < 4.0):
+                        with_cvss[tt][0] += 1
+                    elif (temp_cvss < 7.0):
+                        with_cvss[tt][1] += 1
+                    else:
+                        with_cvss[tt][2] += 1
+
+        # Ignore pkgs with less than one incident, should not happen..
+        if stats[1] < 1:
+            return
+
+        prev_date = 0
+        weight = 0
+
+        dates = sorted(cvestats, key=cvestats.get)
+        try:
+            stats[0] = dates[0]
+        except IndexError:
+            print(pkg + str(dates))
+            stats[0] = 0
+
+        count = sum(cvestats.values())
+
+        print(pkg + ' ' + str(count))
+
+        #    pkg_with_cvss[pkg] = with_cvss
+        self.format_data(pkg, with_cvss, pkg_with_cvss, True)
+
+        self.format_data(pkg, cvestats, src2month, False)
+
+    def format_data(self, pkg, cvestats, src2month, cvss):
+
+        x = []
+        y = []
+        monthyear = []
+        year = []
+
+        temp_items = list(cvestats.items())
+        items = []
+
+        for data_dict in temp_items:
+            if isinstance(data_dict[0], str):
+                tmpx = (parser.parse(data_dict[0]))
+            else:
+                tmpx = data_dict[0]
+            x.append(tmpx)
+
+            try:
+                tmpy = int(data_dict[1])
+            except TypeError:
+                tmpy = data_dict[1]
+            y.append(tmpy)
+            items.append((tmpx, tmpy))
+
+        items.sort(key=lambda tup: tup[0])
+
+        for i in range(2000, 2018):
+            temp = []
+            for j in range(12):
+                if cvss:
+                    temp.append([0, 0, 0])
+                else:
+                    temp.append(0)
+            monthyear.append(temp)
+
+        for i in range(len(x)):
+            if cvss:
+                tmp0 = y[i][0]
+                tmp1 = y[i][1]
+                tmp2 = y[i][2]
+                monthyear[x[i].year - 2000][x[i].month - 1][0] += tmp0
+                monthyear[x[i].year - 2000][x[i].month - 1][1] += tmp1
+                monthyear[x[i].year - 2000][x[i].month - 1][2] += tmp2
+            else:
+                monthyear[x[i].year - 2000][x[i].month - 1] += y[i]
+
+        months_list = [item for sublist in monthyear for item in sublist]
+
+        if not cvss:
+            temp_months = np.zeros(len(months_list))
+            i = 0
+            for element in months_list:
+                temp_months[i] = np.float32(element)
+                i += 1
+
+            src2month[pkg] = temp_months
+        else:
+            src2month[pkg] = months_list
+
+        return
+
+
+
+
+
+
+
+
+

vendors/debian/cache/src2monthDLA

@@ -0,0 +1 @@
+{}

vendors/debian/cache/state

@@ -0,0 +1 @@
+{"Sha1Sums": "", "next_fsa": 0, "vendor": "debian", "Packages": "", "cache_dir": "cache/state", "Sources": "", "next_adv": 4021}

config → vendors/debian/config_default.txt

@@ -14,6 +14,8 @@ arch = amd64
 first_dsa = 11
 first_usn = 1
 first_fsa = 1
+blacklist = DSA-1975, DSA-2360, DSA-2134, DSA-3043, DSA-3156
+
 
 [TRUST]
-lambda = 36
+lambda = 36

vendors/debian/src_name_unifications.txt

@@ -0,0 +1,130 @@
+mozilla-firefox->firefox-esr
+iceweasel->firefox-esr
+mozilla->firefox-esr
+mozilla-thunderbird->icedove
+thunderbird->icedove
+squid\d->squid3
+squid->squid3
+tk8.*->tk8
+xpdf-i->xpdf
+zope*->zope-common
+librmagick-ruby->ruby-rmagick
+libcompass-ruby->ruby-compass
+bio-ruby->ruby-bio
+request-tracker*->request-tracker4
+perl-5.005->perl
+openldap*->openldap
+openoffice*->libreoffice
+nsd3->nsd
+network-manager/network-manager-applet->network-manager
+nagios3->nagios
+nagios2->nagios
+^mysql-\d*->mysql-transitional
+linux-2.6*->linux
+linux-kernel-alpha->linux
+linux-kernel-i386->linux
+libmusicbrainz*->libmusicbrainz
+libgtop1->libgtop2
+libgd1->libgd2
+libast*->libast2
+libmozjs0d->libast
+^kernel-source*->linux
+^kernel-patch*->linux
+^kernel-source-2.4.*->linux
+^kernel-image-2.2.*->linux
+^kernel-image*->linux
+^kernel-patch-*->linux
+kernel-patch-benh->linux
+kernel-patch-vserver->linux
+^kernel-source*->linux
+gnutls*->gnutls28
+gallery2->gallery
+firebird*->firebird3.0
+fltk1.1->fltk1.3
+fox1.4->fox1.6
+exim-tls->exim4
+epic4->epic
+drupal\d->drupal7
+dhcp->dhcpcd5
+cyrus-sasl->cyrus-sasl2
+^cyrus-imapd.*->cyrus-imapd
+^kolab-cyrus-imapd.*->cyrus-imapd
+cfengine->cfengine2
+bind->bind9
+apache->apache2
+horde\d->php-horde
+mediawiki*->mediawiki
+ffmpeg-debian->ffmpeg
+xserver-xorg->xorg-server
+xfree86-1->xorg-server
+xfree86v3->xorg-server
+xfree86->xorg-server
+xfree86->xorg-server
+xorg->xorg-server
+typo3->typo3-src
+lvm10->lvm2
+cupsys->cups
+ethereal->wireshark
+libboost1.42->libboost
+cinelerra-cv->cinelerra
+mplayer-dmo->mplayer
+libcap->libgda2
+xkb-data-legacy->xkeyboard-config
+boost-defaults->boost
+xen-3->xen
+xen-utils->xen
+kde-icons-gorilla->kde-icons-korilla
+libcgi-application-extra-plugin-bundle-perl->libcgi-application-plugins-perl
+^openssl*->openssl
+^tomcat\d->tomcat8
+^tomcat\d.\d$->tomcat8
+^libgda\d->libgda
+^readline\d->readline6
+^libwnck\d->libwnck
+^xerces-c\d->xerces-c
+kde-icons-gorilla->kde-icons-korilla
+kde4libs->kdelibs
+libcgi-application-extra-plugin-bundle-perl->libcgi-application-plugins-perl
+^libticalcs\d->libticals
+^libtifiles\d->libtifiles
+^db\d.\d$->db4.8
+^gcc-.*->gcc
+^automake\d+.*->automake
+^sun-java\d->sun-java6
+^open-jdk\d->open-jdk7
+^mbrola-es\d->mbrola-es
+^mgltools-.*->mgltools
+^coin\d$->coin
+^adobereader-\.*->adobereader
+^picon-\.*->picon
+^nvidia-graphics-drivers\.*->nvidia-graphics-drivers
+^boost\d\.\d\d->boost
+^llvm-\d.\d->llvm
+^octave\d.\d->octave
+^libjibx\d.\d-java->libjibx-java
+^emacs2\d->emacs2
+^emacs2\d-non-dfsg->emacs2
+^libupnp\d->libupnp
+^python\d.\d->python3.5
+^python\d->python3.5
+^postgresql-\d*->postgresql-9.6
+^ruby\d.\d->ruby2.3
+^ruby->ruby2.3
+^mariadb-10.*->mariadb-10.1
+^ruby-actionpack*->rails
+^ruby-activerecord*->rails
+^librack-ruby->ruby-rack
+^libopenssl-ruby->ruby-defaults
+krb4->krb5
+ssh-krb5->openssh
+ssh->openssh
+qemu-kvm->qemu
+kvm->qemu
+phpbb2->phpbb3
+libpng->libpng1.6
+eglibc->glibc
+gnupg->gnupg2
+xine-lib*->xine-lib-1.2
+kfreebsd-\d*->kfreebsd-10
+pdfkit*->pdfkit
+gforge->fusionforge