major additions

bounties.py

@@ -0,0 +1,35 @@
+import json
+
+def main():
+    data = dict()
+    with open("reports.json","r") as fp:
+        data = json.load(fp)
+
+    reports_id = dict()
+    reports_team = dict()
+    teams = []
+    sum_team = dict()
+
+    flag = True
+    for chunk in data:
+        for page_id in chunk:
+            for report in chunk[page_id]:
+                reports_id[report['id']] = report
+                team = report['team']['handle']
+                if team in reports_team:
+                    reports_team[team].append(report)
+                else:
+                    teams.append(team)
+                    reports_team[team] = []
+                    reports_team[team].append(report)
+
+    for team in reports_team:
+        sum_team[team] = len(reports_team[team])
+
+    with open("reports_team.json", "w") as fp:
+        json.dump(reports_team, fp)
+    with open("sum_team.json", "w") as fp:
+        json.dump(sum_team, fp)
+
+if __name__ == "__main__":
+    main()

fixcwes.py

@@ -0,0 +1,82 @@
+import csv
+import re
+import json
+
+def find_roots(cwes_parents, root_list):
+
+    roots = dict()
+
+    for cwe in cwes_parents:
+        ancestor_list = cwes_parents[cwe]
+        roots[cwe] = recursive_ancestor(cwes_parents, ancestor_list, root_list)
+
+    return(roots)
+
+def recursive_ancestor(cwes_parents,ancestor_list, root_list):
+    new_ancestor_list=[]
+    for ancestor in ancestor_list:
+        try:
+            print(ancestor)
+            new_ancestor_list += cwes_parents[ancestor]
+        except KeyError:
+            print('We have a problem with the following ancestor:')
+            print(ancestor)
+            print('#'*80)
+
+
+    flag = True
+    for i in new_ancestor_list:
+        if i not in root_list:
+            flag = False
+            break
+    
+    if flag:
+        return(new_ancestor_list)
+    else:
+        # Yes it is tail-recursive
+        return(recursive_ancestor(cwes_parents,new_ancestor_list, root_list))
+
+def ret_roots(cwe):
+    roots = dict()
+    with open("cwe_roots.json",'r') as fp:
+        roots = json.load(fp)
+
+    try:
+        return(roots[cwe.split('-')[1]])
+    except KeyError:
+        return([0])
+    except IndexError:
+        return([0])
+    except AttributeError:
+        return([0])
+
+
+def main():
+    cwes_parents = dict()
+    cwes_roots = dict()
+    with open("1000.csv","r") as csvfile:
+        spamreader = csv.reader(csvfile, delimiter=',', quotechar='"')
+        for row in spamreader:
+            s = row[6]
+            if row[0]=='732':
+                print(s)
+            m = re.findall(r'::NATURE:ChildOf:CWE ID:([0-9]*):VIEW ID:1000',s)
+            try:
+                #print(row[0],', ', row[6])
+                #print(m.group(1))
+                cwes_parents[row[0]] = m
+                print(m)
+            except AttributeError:
+                print(s)
+    root_list = ['682', '118', '330', '435', '664', '691', '693', '697', '703', '707', '710' ]
+
+    for cwe in root_list:
+        cwes_parents[cwe] = [cwe]
+
+    cwe_roots = find_roots(cwes_parents, root_list)
+    print(cwe_roots)
+    with open("cwe_roots.json","w") as fp:
+        json.dump(cwe_roots,fp)
+
+if __name__ == "__main__":
+    main()

laplace.py

@@ -0,0 +1,20 @@
+import math
+import sys
+
+def laplace_test(ttr, tend):
+    r = len(ttr)
+    s = 0
+    for ti in ttr:
+        s += ti - (tend/2)
+    z = (math.sqrt(12*r)*s)/(r*tend)
+    return(z)
+
+def main():
+    time_end = float(sys.argv[1])
+    time_list = [float(i) for i in sys.argv[2:]]
+    print(laplace_test(time_list, time_end))
+    return
+
+
+if __name__ == "__main__":
+    main()

laplace_tests.py

@@ -0,0 +1,35 @@
+from load_data import load_DBs
+class Mydata:
+    def __init__(self, load):
+        if load:
+            (self.dsatable, self.src2dsa, self.dsa2cve, self.cvetable, self.src2month, self.src2sloccount, self.src2pop, self.src2deps, self.pkg_with_cvss) = load_DBs()
+        else:
+            print('no load command given')
+
+def main():
+    data = Mydata(True)
+    print('Data loading done')
+    #
+    years = 18
+    # 2001-2018
+    
+    i = calc_laplace(data, years)
+    i.calculate()
+
+class calc_laplace:
+    def __init__(self,data,years):
+        self. src2dsa = data.src2dsa
+        self.dsa2cve = data.dsa2cve
+        self.cvetable = data.cvetable
+        self.pkg_with_cvss = data.pkg_with_cvss
+        self.95percent = 1.96
+        self.90percent = 1.64
+
+    def laplace_wheezy(self):
+        pass
+
+    def laplace_package(self, pkg):
+        pass
+
+if __name__ == "__main__":
+    main()

load_data.py

@@ -20,6 +20,7 @@ def load_DBs():
     src2pop = dict()
     src2deps = dict()
     pkg_with_cvss = dict()
+    src2cwe = dict()
 
     cache = config['DIR']['cache_dir']
     
@@ -88,4 +89,11 @@ def load_DBs():
     except (IOError, ValueError):
         print('read cache src2pop failed!! Maybe first run of the system?')
     
-    return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, pkg_with_cvss)
+    src2cwe = cache + 'src2cwe'
+    try:
+        with open(src2cwe) as fp:
+            src2cwe = json.load(fp)
+    except (IOError, ValueError):
+        print('read cache src2cwe failed!! Maybe first run of the system?')
+    
+    return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, pkg_with_cvss, src2cwe)

plot_functions.py

@@ -4,6 +4,8 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pickle
 import vendors.debian.DLAmine as dla
+import json
+import csv
 
 class Plotter:
     def __init__(self, src2month, src2sloccount, pkg_with_cvss, years):
@@ -13,9 +15,14 @@ class Plotter:
         self.years = years
         self.src2month_temp = dict()
         self.src2month_loc = dict()
+        self.l=self.m=self.h=self.udef=0
     
     ## Plot sum of vulnerabilities. Can filter out by severity using the booleans low, med, high, undefined
     def plot_all_severity(self, l, m, h, udef):
+        self.l = l
+        self.m = m
+        self.h = h
+        self.udef = udef
         for i in self.pkg_with_cvss:
             self.src2month_temp[i]=[]
             for j in range(len(self.src2month[i])):
@@ -169,6 +176,11 @@ class WheezyPloter:
     def __init__(self, plotter):
         self.plotter = plotter
         self.yearsx = []
+        self.l = plotter.l
+        self.m = plotter.m
+        self.h = plotter.h
+        self.udef = plotter.udef
+
 
     def plot_wheezy_lts(self):
         quarter_num = self.plotter.years*4
@@ -176,12 +188,69 @@ class WheezyPloter:
         try:
             with open("DLA_sum.txt","rb") as fp:
                 ltslist = pickle.load(fp)
+            with open("src2month_DLA.txt","rb") as fp:
+                src2monthDLAs = pickle.load(fp)
+            with open("DLA_src2month.json","r") as fp:
+                src2monthDLA = json.load(fp)
+            with open("DLA_withcvss.json","r") as fp:
+                self.src2monthDLA_cvss = json.load(fp)
+                # Fix this so it can compute when required
+                #dla.permonthDLA(src2monthDLAs)
+            with open("1000.csv","r") as csvfile:
+                spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
+
         except IOError:
             ltslist = dla.getDLAs()
+            with open("src2month_DLA.txt","rb") as fp:
+                src2monthDLAs = pickle.load(fp)
+            dla.permonthDLA(src2monthDLAs)
+            return self.plot_wheezy_lts()
         
         ## Plot for wheezy
         quarter_sum = [0] * quarter_num
-        totalLTS = [0] * (14 * 12) + ltslist
+
+        DLA_temp=dict()
+        
+        ## Fix src2monthDLA_cvss
+        for i in self.src2monthDLA_cvss:
+            temp_list = []
+            for j in self.src2monthDLA_cvss[i]:
+                temp_list += j
+            self.src2monthDLA_cvss[i] = temp_list
+
+        ## Fix ltslist according to severity
+        for i in self.src2monthDLA_cvss:
+            DLA_temp[i]=[]
+            for j in range(len(self.src2monthDLA_cvss[i])):
+                num_low = self.src2monthDLA_cvss[i][j][0]
+                num_med = self.src2monthDLA_cvss[i][j][1]
+                num_high = self.src2monthDLA_cvss[i][j][2]
+                num_udef = self.src2monthDLA_cvss[i][j][3]
+                tempp = 0
+                if self.l:
+                    tempp += num_low
+                if self.m:
+                    tempp += num_med
+                if self.h:
+                    tempp += num_high
+                if self.udef:
+                    tempp += num_udef
+                DLA_temp[i].append(tempp)
+
+        
+        ltslist = []
+
+        for m in range((self.plotter.years+1)*12):
+            s = 0
+            #print(m)
+            for i in DLA_temp:
+                s += DLA_temp[i][m]
+            ltslist.append(s)
+
+
+        totalLTS = ltslist
+        plt.bar([i for i in range(len(ltslist))],ltslist)
+        plt.show()
 
         quartersx = []
         for i in range(1,self.plotter.years+1):

plot_types.py

@@ -0,0 +1,50 @@
+import matplotlib.pyplot as plt
+
+class TypePlotter:
+    def __init__(self, data, years):
+        self.src2dsa = data.src2dsa
+        self.dsa2cve = data.dsa2cve
+        self.cvetable = data.cvetable
+        self.pkg_with_cvss = data.pkg_with_cvss
+        self.src2cwe = data.src2cwe
+        self.years = years
+
+    def plot_types(self):
+
+        src2cwe_sum = []
+        for i in range(0, 12*self.years):
+            src2cwe_sum.append([0]*12)
+        
+        for pkg in self.src2cwe:
+            for i in range(len(self.src2cwe[pkg])):
+                for j in range(len(self.src2cwe[pkg][i])):
+                    src2cwe_sum[i][j] += self.src2cwe[pkg][i][j]
+
+        src2cwe_sumofsums = [0]*12
+
+        for month in src2cwe_sum:
+            for j in range(len(month)):
+                src2cwe_sumofsums[j] += month[j]
+
+
+        print(src2cwe_sumofsums)
+
+        cwe2month = dict()
+        for j in range(12):
+            cwe2month[j] = []
+
+        for month in src2cwe_sum:
+            for j in range(len(month)):
+                cwe2month[j].append(month[j])
+
+        print(cwe2month[1])
+        print(sum(cwe2month[1]))
+
+        for i in range(12):
+            binned = []
+            for j in range(self.years*3):
+                binned.append(sum(cwe2month[i][4*j:4*j+4]))
+            plt.plot(binned)
+            plt.show()
+
+        return 0

types.py

@@ -1,10 +1,11 @@
 from load_data import load_DBs
 from plot_functions import plot_all
+from plot_types import TypePlotter
 
 class Mydata:
     def __init__(self, load):
         if load:
-            (self.dsatable, self.src2dsa, self.dsa2cve, self.cvetable, self.src2month, self.src2sloccount, self.src2pop, self.src2deps, self.pkg_with_cvss) = load_DBs()
+            (self.dsatable, self.src2dsa, self.dsa2cve, self.cvetable, self.src2month, self.src2sloccount, self.src2pop, self.src2deps, self.pkg_with_cvss, self.src2cwe) = load_DBs()
         else:
             print('no load command given')
 
@@ -12,6 +13,12 @@ def main():
     data = Mydata(True)
     print('Done')
     i = plot_all(data.src2month, data.src2sloccount, data.pkg_with_cvss)
+    #
+    years = 19
+    # 2001-2018
+    
+    j = TypePlotter(data, years)
+    j.plot_types()
 
 if __name__ == "__main__":
     main()

vendors/debian/DLAmine.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
-from DebianAdvisory import DebianAdvisory as da
+from .DebianAdvisory import DebianAdvisory as da
+from .CVEParse import CVEParse as cveparse
 import re
 import datetime
 from html.parser import HTMLParser
@@ -10,6 +11,7 @@ import urllib.request
 import logging, sys
 import pickle
 import json
+from fixcwes import ret_roots
 
 
 def getDLAs():
@@ -44,13 +46,17 @@ def getDLAs():
                         dlas.append([dla, year, month])
                         
                         p1 = re.compile('Package.*: .*')
-                        p2 = re.compile('CVE-[0-9]{4}-[0-9]{4}')
+                        p2 = re.compile('CVE-[0-9]{4}-[0-9]*')
                         (pkg, cves) = parseDLAhtml(dla, p1, p2)
                         pkg = fixURL(url, pkg)
 
                         if pkg:
-                            src2monthDLAs[pkg] = (cves, [year,month])
                             totalDLAs[str(year) + ',' + str(month)] += cves
+                            try:
+                                src2monthDLAs[pkg].append((cves, [year,month]))
+                            except KeyError:
+                                src2monthDLAs[pkg] = []
+                                src2monthDLAs[pkg].append((cves, [year,month]))
 
                 except urllib.error.HTTPError as err:
                     if (i>1):
@@ -68,11 +74,87 @@ def getDLAs():
     with open("DLA_sum.txt","wb") as fp:
         pickle.dump(totalDLAsList,fp)
     
-    with open("src2month_DLA.txt","w") as fp:
+    with open("src2month_DLA.txt","wb") as fp:
+        pickle.dump(src2monthDLAs,fp)
+    
+    with open("src2month_DLA.json","w") as fp:
         json.dump(src2monthDLAs,fp)
 
     return(totalDLAsList)
 
+def permonthDLA(src2monthDLAs):
+    client = MongoClient()
+    out = dict()
+    out_cvss = dict()
+    out_cwe = dict()
+    for pkg in src2monthDLAs:
+        (out[pkg], out_cvss[pkg], out_cwe[pkg]) = perPackage(pkg, src2monthDLAs[pkg], out, out_cvss, client)
+        #out_cwe[pkg] = perPackage_cwe(pkg, src2monthDLAs[pkg])
+        with open("DLA_src2month.json","w") as fp:
+            json.dump(out,fp)
+        
+        with open("DLA_withcvss.json","w") as fp:
+            json.dump(out_cvss,fp)
+        #
+        with open("DLA_withcwe.json","w") as fp:
+            json.dump(out_cwe,fp)
+
+def perPackage(pkg, dlas, cvss, out, client): 
+    root_list = ['682', '118', '330', '435', '664', '691', '693', '697', '703', '707', '710' ]
+    monthyear = []
+    monthyear_cvss = []
+    monthyear_cwe = []
+    haveseen = dict()
+    for i in range(2000,2019):
+        temp = []
+        temp_cvss = []
+        temp_cwe = []
+        for j in range(12):
+            temp.append(0)
+            temp_cvss.append([0,0,0,0])
+            temp_cwe.append([0]*12)
+        monthyear.append(temp)
+        monthyear_cvss.append(temp_cvss)
+        monthyear_cwe.append(temp_cwe)
+    
+    for dla in dlas:
+        for cve_id in dla[0]:
+            if cve_id in haveseen:
+                continue
+            else:
+                haveseen[cve_id] = 1
+            cve = cveparse.fetchCVE(cve_id, client)
+            (cve_date, cve_base, cve_impact, cve_exploit, cwe) = cveparse.parseCVE(cve_id, cve)
+            new_year = dla[1][0]
+            new_month = dla[1][1]
+            if (cve_date.year<new_year) or (cve_date.year==new_year and cve_date.month<new_month):
+                new_year=cve_date.year
+                new_month=cve_date.month
+            try:
+                cve_base = float(cve_base)
+            except TypeError:
+                cve_base = -1.0
+            monthyear[new_year-2000][new_month-1] += 1
+            if (cve_base < 0.0):
+                monthyear_cvss[new_year-2000][new_month-1][3] += 1
+            elif (cve_base < 4.0):
+                monthyear_cvss[new_year-2000][new_month-1][0] += 1
+            elif (cve_base < 7.0):
+                monthyear_cvss[new_year-2000][new_month-1][1] += 1
+            else:
+                monthyear_cvss[new_year-2000][new_month-1][2] += 1
+            
+            for i in ret_roots(cwe):
+                if i == 0:
+                    monthyear_cwe[new_year-2000][new_month-1][11] += 1
+                    print('Problem with cve: ', cve_id)
+                    continue
+                for j in range(len(root_list)):
+                    if i == root_list[j]:
+                        monthyear_cwe[new_year-2000][new_month-1][j] += 1
+
+    return(monthyear, monthyear_cvss, monthyear_cwe)
+
 
 def parseDLAhtml(dla, p1, p2):
     
@@ -81,13 +163,15 @@ def parseDLAhtml(dla, p1, p2):
     if pkg:
         print(pkg.group(0))
         pkg = pkg.group(0).split(':',1)[1].strip()
+        # Deal witg the different versions also here...
+        pkg=da.unifySrcName(pkg)
         print(pkg)
     else:
         print(dla)
     
-    p2 = re.compile('CVE-[0-9]{4}-[0-9]{4}')
     cves = re.findall(p2, dla)
     cves = list(set(cves))
+
     
     return (pkg, cves)
 

vendors/debian/DebianModel.py

@@ -12,6 +12,7 @@ from .CVEParse import CVEParse
 from ..VendorModel import VendorModel
 from .CSVReader import CSVReader
 from .Tests import Tests
+from fixcwes import ret_roots
 
 class DebianModel(VendorModel):
     """
@@ -35,6 +36,9 @@ class DebianModel(VendorModel):
         self.src2deps = dict()
         self.pkg_with_cvss = dict()
         self.src2sum = dict()
+        self.src2monthDLA = dict()
+        self.pkg_with_cvss_DLA = dict()
+        self.src2cwe = dict()
 
 
         ## config
@@ -99,6 +103,9 @@ class DebianModel(VendorModel):
         self.src2sloccount = self.load_single_db_from_cache('src2sloccount')
         self.src2pop = self.load_single_db_from_cache('src2pop')
         self.pkg_with_cvss = self.load_single_db_from_cache('pkg_with_cvss')
+        self.src2monthDLA = self.load_single_db_from_cache('src2monthDLA')
+        self.pkg_with_cvss_DLA = self.load_single_db_from_cache('plg_with_cvss_DLA')
+        self.src2cwe = self.load_single_db_from_cache('src2cwe')
 
     def load_single_db_from_cache(self, file_name):
         cache_dir = os.path.join(self.module_path, self.config['DIR']['cache_dir'])
@@ -118,8 +125,12 @@ class DebianModel(VendorModel):
         self.store_db_single('src2sloccount', self.src2sloccount)
         self.store_db_single('src2pop', self.src2pop)
         self.store_db_single('pkg_with_cvss', self.pkg_with_cvss)
+        self.store_db_single('src2monthDLA', self.src2monthDLA)
+        self.store_db_single('pkg_with_cvss_DLA', self.pkg_with_cvss_DLA)
+        self.store_db_single('src2cwe', self.src2cwe)
 
         # src2month needs special handling
+        # Check later if the same is true for other dicts
         cache_src2month = os.path.join(self.module_path, self.config['DIR']['cache_dir'], 'src2month')
         int_list = dict()
 
@@ -235,11 +246,6 @@ class DebianModel(VendorModel):
         return state, err
 
     def processCVEs(self, srcpkg, now):
-        """
-        compute and store MTBF, MTBR and Scores of each src pkg
-        output: %src2mtbf
-        (srcpkg=> ())
-        """
 
         stats = [now, 0, 0, 0, 0, 0, 0]
         cvestats = dict()
@@ -248,6 +254,9 @@ class DebianModel(VendorModel):
         ## keep track of the number of low-medium-high severity vulnerabilities
         ## TODO see how cvss affects vulnerability prediction - if some packages show patterns
         with_cvss = dict()
+        ## Keep track of the vulnerability types
+        with_cwe = dict()
+        root_list = ['682', '118', '330', '435', '664', '691', '693', '697', '703', '707', '710' ]
 
         ## To eliminate duplicate cves
         haveseen = dict()
@@ -266,9 +275,8 @@ class DebianModel(VendorModel):
                         cvestats[tt] = 1
             stats[1] += 1
 
-        ## Date at the moment taken from CVE? - not sure.
-
         haveseen = dict()
+        haveseen2 = dict()
 
         ## with_cvss = (date: number low, number med, number high, number undefined)
         for dsa_id in self.src2dsa[srcpkg]:
@@ -304,6 +312,41 @@ class DebianModel(VendorModel):
                         else:
                             with_cvss[tt][2] += 1
 
+        ## with_cwe = (date: 11*[root type])
+        for dsa_id in self.src2dsa[srcpkg]:
+            for cve_id in self.dsa2cve[str(dsa_id)]:
+                tt = self.cvetable[cve_id][0]
+                try:
+                    temp_cwe = self.cvetable[cve_id][5]
+                except TypeError:
+                    print(cve_id)
+                    continue
+
+                if cve_id in haveseen2:
+                    continue
+                else:
+                    haveseen2[cve_id] = 1
+                    if tt in with_cwe:
+                        for i in ret_roots(temp_cwe):
+                            if i == 0:
+                                with_cwe[tt][11] += 1
+                                print('Problem with cve: ', cve_id)
+                            for j in range(len(root_list)):
+                                if i == root_list[j]:
+                                    with_cwe[tt][j] += 1
+                    else:
+                        with_cwe[tt] = [0]*12
+                        for i in ret_roots(temp_cwe):
+                            if i == 0:
+                                with_cwe[tt][11] += 1
+                                print('Problem with cve: ', cve_id)
+                            for j in range(len(root_list)):
+                                print('This is the with_cwe')
+                                if i == root_list[j]:
+                                    with_cwe[tt][j] += 1
+                                    print('This is the with_cwe')
+                                    print(with_cwe)
+
         # Ignore pkgs with less than one incident, should not happen..
         if stats[1] < 1:
             return
@@ -321,6 +364,48 @@ class DebianModel(VendorModel):
 
         self.format_data(srcpkg, cvestats, self.src2month, False)
 
+        self.format_cwes(srcpkg, with_cwe)
+
+#########################################################################
+    def format_cwes(self, srcpkg, with_cwe):
+        x = []
+        y = []
+        monthyear = []
+        year = []
+
+        temp_items = list(with_cwe.items())
+        items = []
+
+        for cwe_dict in temp_items:
+            if isinstance(cwe_dict[0], str):
+                tmpx = (parser.parse(cwe_dict[0]))
+            else:
+                tmpx = cwe_dict[0]
+            x.append(tmpx)
+            
+            tmpy = cwe_dict[1]
+            
+            y.append(tmpy)
+            items.append((tmpx, tmpy))
+
+        items.sort(key=lambda tup: tup[0])
+
+        for i in range(2000,2019):
+            temp = []
+            for j in range(12):
+                temp.append([0]*12)
+            monthyear.append(temp)
+
+        for i in range(len(x)):
+            for j in range(len(y[i])):
+                monthyear[x[i].year - 2000][x[i].month - 1][j] += y[i][j]
+
+        months_list = [item for sublist in monthyear for item in sublist]
+
+        self.src2cwe[srcpkg] = months_list
+
+
+
     def format_data(self, pkg, cvestats, src2temp, cvss):
         x = []
         y = []
@@ -355,9 +440,6 @@ class DebianModel(VendorModel):
                     temp.append(0)
             monthyear.append(temp)
 
-        if pkg=='linux':
-            print(x)
-
         for i in range(len(x)):
             if cvss:
                 tmp0 = y[i][0]

vendors/debian/cache/pkg_with_cvss_DLA

@@ -0,0 +1 @@
+{}

vendors/debian/cache/src2monthDLA

@@ -0,0 +1 @@
+{}

vendors/debian/cache/state

@@ -1 +1 @@
-{"cache_dir": "cache/", "vendor": "debian", "next_adv": 4324, "next_fsa": 0, "Packages": "", "Sources": "", "Sha1Sums": ""}
+{"cache_dir": "cache/", "vendor": "debian", "next_adv": 4335, "next_fsa": 0, "Packages": "", "Sources": "", "Sha1Sums": ""}