added dependency data

apt-sec.py

@@ -23,6 +23,7 @@ import plotly.graph_objs as go
 import machine_learning as ml
 import lstm_reg as lstm
 import metadata as meta
+import deps
 import psycopg2
 
 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
@@ -114,6 +115,7 @@ def load_DBs():
     src2month = dict()
     src2sloccount = dict()
     src2pop = dict()
+    src2deps = dict()
 
     cache = config['DIR']['cache_dir']
     
@@ -146,6 +148,14 @@ def load_DBs():
     except (IOError, ValueError):
         print('read cache cvetable failed!! Maybe first run of the system?')
 
+    cache_src2deps = cache + 'src2deps'
+    try:
+        with open(cache_src2deps) as fp:
+            src2deps = json.load(fp)
+    except (IOError, ValueError):
+        print('read cache src2deps failed!! Maybe first run of the system?')
+
+
     cache_src2month = cache + 'src2month'
     try:
         with open(cache_src2month) as fp:
@@ -167,7 +177,7 @@ def load_DBs():
     except (IOError, ValueError):
         print('read cache src2pop failed!! Maybe first run of the system?')
     
-    return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop)
+    return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps)
 
 
 ###############################################################################
@@ -179,7 +189,7 @@ def myconverter(o):
         return o.astype(int)
 ###############################################################################
 ## save to files
-def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop):
+def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps):
     cache = config['DIR']['cache_dir']
     
     cache_dsatable = cache + 'dsatable'
@@ -230,6 +240,14 @@ def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src
         print('write cache src2pop failed!! Fatal error')
         sys.exit(1)
     
+    cache_src2deps = cache + 'src2deps'
+    try:
+        with open(cache_src2deps, 'w') as fp:
+            json.dump(src2deps, fp, default = myconverter)
+    except IOError:
+        print('write cache src2deps failed!! Fatal error')
+        sys.exit(1)
+
     cache_src2month = cache + 'src2month'
     int_list = dict()
     
@@ -735,6 +753,12 @@ def getpop(src2dsa, src2pop):
     return
 
 
+###############################################################################
+## get dependencies of a given source
+def getdeps(src2dsa, src2deps):
+    for srcpkg in src2dsa:
+        deps.getdeps(srcpkg, src2deps)
+
 
 ###############################################################################
 ## print some meta-info on internal data
@@ -847,6 +871,7 @@ src2dsa = dict()
 dsa2cve = dict()
 cvetable = dict()
 src2month = dict()
+src2deps = dict()
 pkg_with_cvss = dict()
 src2sloccount = dict()
 src2pop = dict()
@@ -860,13 +885,14 @@ state['vendor'] = 'debian'
 #    os.makedirs(d)
 
 if action == 'update':
-    (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop) = load_DBs()
+    (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = load_DBs()
 #    loadsha1lists()
     aptsec_update(state,config, dsatable, client, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss)
 #    save_sha1lists()
-    getslocs(src2dsa, src2sloccount)
+#    getslocs(src2dsa, src2sloccount)
     getpop(src2dsa, src2pop)
-    save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop)
+    getdeps(src2dsa, src2deps)
+    save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps)
     save_state(state)
 #    ml.predict(src2month)
 #    lstm.predict(src2month)

deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/python3
+import apt
+
+def getdeps(srcpkg, src2deps):
+    
+    src2deps[srcpkg] = []
+
+    cache = apt.Cache()
+    counter = 0
+
+    for binpkg in cache:
+        if binpkg.versions[0].source_name == srcpkg:
+            deps = binpkg.versions[0].get_dependencies('PreDepends', 'Depends')
+            for dep in deps:
+                try:
+                    srcdep = cache[dep.or_dependencies[0].name].versions[0].source_name
+                except KeyError:
+                    srcdep = 'package does not exist currently'
+
+                if not (srcdep in src2deps[srcpkg]):
+                    src2deps[srcpkg].append(srcdep)
+    return 0

test.py

@@ -0,0 +1,18 @@
+#!/usr/bin/python3
+import apt
+
+def getdeps(srcpkg, src2deps):
+    
+    src2deps[srcpkg] = []
+
+    cache = apt.Cache()
+    counter = 0
+
+    for binpkg in cache:
+        if binpkg.versions[0].source_name == srcpkg:
+            deps = binpkg.versions[0].getdependencies('PreDepends', 'Depends')
+            for dep in deps:
+                srcdep = cache[dep.or_dependencies[0].name].source_name
+                if not (srcdep in src2deps[srcpkg]):
+                    src2deps[srcpkg].append(srcdep)
+    return 0