|
@@ -1,4 +1,205 @@
|
|
|
#!/usr/bin/python3
|
|
|
+## New implementation of TrustMiner using python and mongodb
|
|
|
+## Nikos
|
|
|
+
|
|
|
+import sys
|
|
|
+from pymongo import MongoClient
|
|
|
+#mongodb assumes database at default path
|
|
|
import logging, sys
|
|
|
+import configparser
|
|
|
+import json
|
|
|
+import urllib.request
|
|
|
+import datetime
|
|
|
+import debian_advisory
|
|
|
|
|
|
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
|
+
|
|
|
+#load config file as library
|
|
|
+config = configparser.ConfigParser()
|
|
|
+config.read('config_test')
|
|
|
+if config.sections == []:
|
|
|
+ print('configuration file not found\n')
|
|
|
+ sys.exit(1)
|
|
|
+
|
|
|
+#global variables
|
|
|
+secperday = 60*60*24
|
|
|
+now = datetime.datetime.now()
|
|
|
+verbosity = 1
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+
|
|
|
+## logging
|
|
|
+# 1 fatal errors
|
|
|
+# 2 errors
|
|
|
+# 3 note
|
|
|
+# 4 trace
|
|
|
+# 5 debug
|
|
|
+
|
|
|
+def msg(lvl,msg):
|
|
|
+ if lvl <= int(config['LOG']['loglevel']):
|
|
|
+ print(msg)
|
|
|
+
|
|
|
+def debug(msg):
|
|
|
+ msg(5, msg)
|
|
|
+# Need to see if this is necessary
|
|
|
+
|
|
|
+## load state, different from DBs in that we always need it
|
|
|
+def load_state():
|
|
|
+ cache = config['DIR']['cache_dir'] + 'state'
|
|
|
+ err = 0
|
|
|
+ state = dict()
|
|
|
+
|
|
|
+ try:
|
|
|
+ with open(cache) as json_data:
|
|
|
+ state = json.load(json_data)
|
|
|
+ except FileNotFoundError:
|
|
|
+ # Load default state - start from the beginning
|
|
|
+ state['next_adv'] = 0
|
|
|
+ state['next_fsa'] = 0
|
|
|
+ state['Packages'] = ''
|
|
|
+ state['Sources'] = ''
|
|
|
+ state['Sha1Sums'] = ''
|
|
|
+ err += 1
|
|
|
+
|
|
|
+ return (state, err)
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## save state, different from DBs in that we always need it
|
|
|
+def save_state(state):
|
|
|
+ cache = config['DIR']['cache_dir'] + 'state'
|
|
|
+
|
|
|
+ try:
|
|
|
+ with open(cache, 'w') as fp:
|
|
|
+ json.dump(state, fp)
|
|
|
+ except IOError:
|
|
|
+ print('write cache failed!! Fatal error')
|
|
|
+ sys.exit(1)
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## load sha lists :TODO later
|
|
|
+def load_sha1lists():
|
|
|
+ cache = config['DIR']['cache_dir'] + 'state'
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## save sha lists :TODO later
|
|
|
+def save_sha1lists():
|
|
|
+ pass
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## load from files :TODO later
|
|
|
+def load_DBs():
|
|
|
+ pass
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## save to files :TODO later
|
|
|
+def save_DBs():
|
|
|
+ pass
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## Fetch current Packages, Sources and sha1sums files
|
|
|
+## These are needed to find CVE stats by sha1sums/pkg-names
|
|
|
+## Only Sha1Sums is custom generated, others are from Debian.
|
|
|
+## FIXME: Server might do on-the-fly gzip (but should not for bzip2)
|
|
|
+## Return: 1 on success, to signal that new parsing is needed.
|
|
|
+def fetchMeta(filename):
|
|
|
+ urlbase = config['URL']['pkg_base_url']
|
|
|
+ mydir = config['DIR']['cache_dir']
|
|
|
+ bzFile = filename + '.bz2'
|
|
|
+ url = urlbase + bzFile
|
|
|
+
|
|
|
+ logging.info('Checking meta file from ' + url + '\n')
|
|
|
+
|
|
|
+ # Download file
|
|
|
+ urllib.request.urlretrieve(url, mydir + bzfile)
|
|
|
+ # TODO catch exceptions like file not found
|
|
|
+ # TODO check if file has changed, if it is new unpack
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+# Sources and Packages are not completely consistent, esp for debian-multimedia
|
|
|
+# He we store manual mappings for these..
|
|
|
+def addOrphanPkgs(pkg2src):
|
|
|
+ pkg2src['liblame-dev'] = "lame";
|
|
|
+ pkg2src['lame-extras'] = "lame";
|
|
|
+ pkg2src['moonlight'] = "moon";
|
|
|
+ pkg2src['libmoon0'] = "moon";
|
|
|
+ pkg2src['xmms-mp4'] = "xmms2";
|
|
|
+ pkg2src['xmms-mp4'] = "xmms2";
|
|
|
+ pkg2src['lazarus-src-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lazarus-ide-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-qt4-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lazarus-ide-qt4-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-gtk2-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lazarus-ide-gtk2-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-units-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lazarus-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lazarus-doc-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-utils-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['lcl-nogui-0.9.30'] = "lazarus";
|
|
|
+ pkg2src['libx264-65'] = "x264";
|
|
|
+ pkg2src['libx264-114'] = "x264";
|
|
|
+ pkg2src['libx264-60'] = "x264";
|
|
|
+# pkg2src['libmlt3']
|
|
|
+# pkg2src['libgmerlin-avdec0']
|
|
|
+# pkg2src['libxul-dev']
|
|
|
+# pkg2src['libmyth-0.23.1-0']
|
|
|
+# pkg2src['libmpeg3hv']
|
|
|
+# pkg2src['libquicktimehv']
|
|
|
+# pkg2src['libxul0d']
|
|
|
+# pkg2src['acroread-fonts-kor']
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## Parse dpkg Packages file, create map deb-name->pkg-name
|
|
|
+def parsePackages(pkgfile):
|
|
|
+ mydir = cache = config['DIR']['cache_dir']
|
|
|
+ deb2pkg = dict()
|
|
|
+ pkg2virt = dict()
|
|
|
+ virt2pkg = ()
|
|
|
+
|
|
|
+ logging.info('Parsing Packages file...\n')
|
|
|
+ pkgfile = mydir + pkgfile
|
|
|
+
|
|
|
+ #TODO open and parse pkg file
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+## Parse dpkg Sources file, create map pkg-name->src-name
|
|
|
+def parseSources(srcfile)
|
|
|
+ mydir = cache = config['DIR']['cache_dir']
|
|
|
+ checklinecont = 0
|
|
|
+ pkg2src = dict()
|
|
|
+
|
|
|
+ logging.info('Parsing Sources file...\n')
|
|
|
+ srcfile = mydir + srcfile
|
|
|
+
|
|
|
+ #TODO open and parse sources file
|
|
|
+
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+def getSHA1(myhash, collection):
|
|
|
+ return collection.find({"hash": myhash})
|
|
|
+
|
|
|
+
|
|
|
+###############################################################################
|
|
|
+def addSHA1(myhash, deb, src)
|
|
|
+ dic = getSHA1(myhash)
|
|
|
+ thash = dic["hash"]
|
|
|
+ tdeb = dic["deb"]
|
|
|
+ tsrc = dic["src"]
|
|
|
+
|
|
|
+ #TODO insert SHA to database
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+(state, err) = load_state()
|
|
|
+print(state)
|
|
|
+save_state(state)
|
|
|
+
|
|
|
+#client = MongoClient()
|
|
|
+
|
|
|
+#cve_db = client.cvedb
|
|
|
+#collection = db.cves
|
|
|
+#testcvss = collection.find_one({"cvss": 9.3})
|
|
|
+#print(testcvss)
|