|
@@ -23,6 +23,7 @@ import plotly.graph_objs as go
|
|
|
import machine_learning as ml
|
|
|
import lstm_reg as lstm
|
|
|
import metadata as meta
|
|
|
+import deps
|
|
|
import psycopg2
|
|
|
|
|
|
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
@@ -114,6 +115,7 @@ def load_DBs():
|
|
|
src2month = dict()
|
|
|
src2sloccount = dict()
|
|
|
src2pop = dict()
|
|
|
+ src2deps = dict()
|
|
|
|
|
|
cache = config['DIR']['cache_dir']
|
|
|
|
|
@@ -146,6 +148,14 @@ def load_DBs():
|
|
|
except (IOError, ValueError):
|
|
|
print('read cache cvetable failed!! Maybe first run of the system?')
|
|
|
|
|
|
+ cache_src2deps = cache + 'src2deps'
|
|
|
+ try:
|
|
|
+ with open(cache_src2deps) as fp:
|
|
|
+ src2deps = json.load(fp)
|
|
|
+ except (IOError, ValueError):
|
|
|
+ print('read cache src2deps failed!! Maybe first run of the system?')
|
|
|
+
|
|
|
+
|
|
|
cache_src2month = cache + 'src2month'
|
|
|
try:
|
|
|
with open(cache_src2month) as fp:
|
|
@@ -167,7 +177,7 @@ def load_DBs():
|
|
|
except (IOError, ValueError):
|
|
|
print('read cache src2pop failed!! Maybe first run of the system?')
|
|
|
|
|
|
- return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop)
|
|
|
+ return(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps)
|
|
|
|
|
|
|
|
|
###############################################################################
|
|
@@ -179,7 +189,7 @@ def myconverter(o):
|
|
|
return o.astype(int)
|
|
|
###############################################################################
|
|
|
## save to files
|
|
|
-def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop):
|
|
|
+def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps):
|
|
|
cache = config['DIR']['cache_dir']
|
|
|
|
|
|
cache_dsatable = cache + 'dsatable'
|
|
@@ -230,6 +240,14 @@ def save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src
|
|
|
print('write cache src2pop failed!! Fatal error')
|
|
|
sys.exit(1)
|
|
|
|
|
|
+ cache_src2deps = cache + 'src2deps'
|
|
|
+ try:
|
|
|
+ with open(cache_src2deps, 'w') as fp:
|
|
|
+ json.dump(src2deps, fp, default = myconverter)
|
|
|
+ except IOError:
|
|
|
+ print('write cache src2deps failed!! Fatal error')
|
|
|
+ sys.exit(1)
|
|
|
+
|
|
|
cache_src2month = cache + 'src2month'
|
|
|
int_list = dict()
|
|
|
|
|
@@ -735,6 +753,12 @@ def getpop(src2dsa, src2pop):
|
|
|
return
|
|
|
|
|
|
|
|
|
+###############################################################################
|
|
|
+## get dependencies of a given source
|
|
|
+def getdeps(src2dsa, src2deps):
|
|
|
+ for srcpkg in src2dsa:
|
|
|
+ deps.getdeps(srcpkg, src2deps)
|
|
|
+
|
|
|
|
|
|
###############################################################################
|
|
|
## print some meta-info on internal data
|
|
@@ -847,6 +871,7 @@ src2dsa = dict()
|
|
|
dsa2cve = dict()
|
|
|
cvetable = dict()
|
|
|
src2month = dict()
|
|
|
+src2deps = dict()
|
|
|
pkg_with_cvss = dict()
|
|
|
src2sloccount = dict()
|
|
|
src2pop = dict()
|
|
@@ -860,13 +885,14 @@ state['vendor'] = 'debian'
|
|
|
# os.makedirs(d)
|
|
|
|
|
|
if action == 'update':
|
|
|
- (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop) = load_DBs()
|
|
|
+ (dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps) = load_DBs()
|
|
|
# loadsha1lists()
|
|
|
aptsec_update(state,config, dsatable, client, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss)
|
|
|
# save_sha1lists()
|
|
|
- getslocs(src2dsa, src2sloccount)
|
|
|
+# getslocs(src2dsa, src2sloccount)
|
|
|
getpop(src2dsa, src2pop)
|
|
|
- save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop)
|
|
|
+ getdeps(src2dsa, src2deps)
|
|
|
+ save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps)
|
|
|
save_state(state)
|
|
|
# ml.predict(src2month)
|
|
|
# lstm.predict(src2month)
|