Browse Source

started main file in python

Nikolaos Alexopoulos 7 years ago
parent
commit
3fa1448500

BIN
.common-vulnerability-entry.pl.swp → .apt-sec.py.swp


BIN
.common-vulnerability-entry.py.swp → .config_test.swp


BIN
.debian-security-advisory.pl.swp


BIN
.debian-security-advisory.py.swp


BIN
__pycache__/debian_advisory.cpython-35.pyc


+ 1 - 0
apt-sec.conf

@@ -1,4 +1,5 @@
 # 0 = quiet, 1 = fatal errors, 2 = errors, 3 = notice, 4 = trace, 5 = debug
+[DEFAULT]
 loglevel = 4
 
 #dsa_base_url = http://www.debian.org/security/

+ 201 - 0
apt-sec.py

@@ -1,4 +1,205 @@
 #!/usr/bin/python3
+## New implementation of TrustMiner using python and mongodb
+## Nikos
+
+import sys
+from pymongo import MongoClient
+#mongodb assumes database at default path
 import logging, sys
+import configparser
+import json
+import urllib.request
+import datetime
+import debian_advisory
 
 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+#load config file as library
+config = configparser.ConfigParser()
+config.read('config_test')
+if config.sections == []:
+    print('configuration file not found\n')
+    sys.exit(1)
+
+#global variables
+secperday = 60*60*24
+now = datetime.datetime.now()
+verbosity = 1
+
+###############################################################################
+
+## logging
+# 1 fatal errors
+# 2 errors
+# 3 note
+# 4 trace
+# 5 debug
+
+def msg(lvl,msg):
+    if lvl <= int(config['LOG']['loglevel']):
+        print(msg)
+
+def debug(msg):
+    msg(5, msg)
+# Need to see if this is necessary
+
+## load state, different from DBs in that we always need it
+def load_state():
+    cache = config['DIR']['cache_dir'] + 'state'
+    err = 0
+    state = dict()
+
+    try:
+        with open(cache) as json_data:
+            state = json.load(json_data)
+    except FileNotFoundError:
+        # Load default state - start from the beginning
+        state['next_adv'] = 0
+        state['next_fsa'] = 0
+        state['Packages'] = ''
+        state['Sources'] = ''
+        state['Sha1Sums'] = ''
+        err += 1
+
+    return (state, err)
+
+###############################################################################
+## save state, different from DBs in that we always need it
+def save_state(state):
+    cache = config['DIR']['cache_dir'] + 'state'
+    
+    try:
+        with open(cache, 'w') as fp:
+            json.dump(state, fp)
+    except IOError:
+        print('write cache failed!! Fatal error')
+        sys.exit(1)
+
+###############################################################################
+## load sha lists :TODO later
+def load_sha1lists():
+    cache = config['DIR']['cache_dir'] + 'state'
+
+###############################################################################
+## save sha lists :TODO later
+def save_sha1lists():
+    pass
+
+###############################################################################
+## load from files :TODO later
+def load_DBs():
+    pass
+
+###############################################################################
+## save to files :TODO later
+def save_DBs():
+    pass
+
+###############################################################################
+## Fetch current Packages, Sources and sha1sums files
+## These are needed to find CVE stats by sha1sums/pkg-names
+## Only Sha1Sums is custom generated, others are from Debian.
+## FIXME: Server might do on-the-fly gzip (but should not for bzip2)
+## Return: 1 on success, to signal that new parsing is needed.
+def fetchMeta(filename):
+    urlbase = config['URL']['pkg_base_url']
+    mydir = config['DIR']['cache_dir']
+    bzFile = filename + '.bz2'
+    url = urlbase + bzFile
+
+    logging.info('Checking meta file from ' + url + '\n')
+
+    # Download file
+    urllib.request.urlretrieve(url, mydir + bzfile)
+    # TODO catch exceptions like file not found
+    # TODO check if file has changed, if it is new unpack
+
+###############################################################################
+# Sources and Packages are not completely consistent, esp for debian-multimedia
+# He we store manual mappings for these..
+def addOrphanPkgs(pkg2src):
+    pkg2src['liblame-dev'] = "lame";
+    pkg2src['lame-extras'] = "lame";
+    pkg2src['moonlight'] = "moon";
+    pkg2src['libmoon0'] = "moon";
+    pkg2src['xmms-mp4'] = "xmms2";
+    pkg2src['xmms-mp4'] = "xmms2";
+    pkg2src['lazarus-src-0.9.30'] = "lazarus";
+    pkg2src['lazarus-ide-0.9.30'] = "lazarus";
+    pkg2src['lcl-qt4-0.9.30'] = "lazarus";
+    pkg2src['lazarus-ide-qt4-0.9.30'] = "lazarus";
+    pkg2src['lcl-gtk2-0.9.30'] = "lazarus";
+    pkg2src['lazarus-ide-gtk2-0.9.30'] = "lazarus";
+    pkg2src['lcl-units-0.9.30'] = "lazarus";
+    pkg2src['lazarus-0.9.30'] = "lazarus";
+    pkg2src['lazarus-doc-0.9.30'] = "lazarus";
+    pkg2src['lcl-0.9.30'] = "lazarus";
+    pkg2src['lcl-utils-0.9.30'] = "lazarus";
+    pkg2src['lcl-nogui-0.9.30'] = "lazarus";
+    pkg2src['libx264-65'] = "x264";
+    pkg2src['libx264-114'] = "x264";
+    pkg2src['libx264-60'] = "x264";
+#  pkg2src['libmlt3']
+#  pkg2src['libgmerlin-avdec0']
+#  pkg2src['libxul-dev']
+#  pkg2src['libmyth-0.23.1-0']
+#  pkg2src['libmpeg3hv']
+#  pkg2src['libquicktimehv']
+#  pkg2src['libxul0d']
+#  pkg2src['acroread-fonts-kor']
+
+###############################################################################
+## Parse dpkg Packages file, create map deb-name->pkg-name
+def parsePackages(pkgfile):
+    mydir = cache = config['DIR']['cache_dir']
+    deb2pkg = dict()
+    pkg2virt = dict()
+    virt2pkg = ()
+
+    logging.info('Parsing Packages file...\n')
+    pkgfile = mydir + pkgfile
+
+    #TODO open and parse pkg file
+
+###############################################################################
+## Parse dpkg Sources file, create map pkg-name->src-name
+def parseSources(srcfile)
+    mydir = cache = config['DIR']['cache_dir']
+    checklinecont = 0
+    pkg2src = dict()
+
+    logging.info('Parsing Sources file...\n')
+    srcfile = mydir + srcfile
+    
+    #TODO open and parse sources file
+
+
+###############################################################################
+def getSHA1(myhash, collection):
+    return collection.find({"hash": myhash})
+
+
+###############################################################################
+def addSHA1(myhash, deb, src)
+    dic = getSHA1(myhash)
+    thash = dic["hash"]
+    tdeb = dic["deb"]
+    tsrc = dic["src"]
+
+    #TODO insert SHA to database
+
+
+
+
+
+
+(state, err) = load_state()
+print(state)
+save_state(state)
+
+#client = MongoClient()
+
+#cve_db = client.cvedb
+#collection = db.cves
+#testcvss = collection.find_one({"cvss": 9.3})
+#print(testcvss)

+ 1 - 1
common-vulnerability-entry.py

@@ -82,7 +82,7 @@ def parseCVE(cve_id, cve):
     if cve == '':
         # No details means we assume worst-case (highest score, recent bug)
         if re.compile('LOCAL-(.*)').match(cve_id):
-            
+            logging.info('Assuming worst-case ratings for LOCAL CVE ' + cve_id)
 
 
 

+ 19 - 0
config_test

@@ -0,0 +1,19 @@
+[LOG]
+loglevel = 4
+
+[URL]
+dsa_base_url = http://www.debian.org/security
+pkg_base_url = https://freeside.trust.cased.de/apt-sec/mirror
+
+[DIR]
+cache_dir = cache/
+dists_dir = mirror/dists/
+arch = amd64
+
+[DSA]
+first_dsa = 11
+first_usn = 1
+first_fsa = 1
+
+[TRUST]
+lambda = 36

+ 1 - 0
conftest.ini

@@ -0,0 +1 @@
+[DEFAULT]
ServerAliveInterval = 45
Compression = yes
CompressionLevel = 9
ForwardX11 = yes

[bitbucket.org]
User = hg

[topsecret.server.com]
Port = 50022
ForwardX11 = no

+ 1 - 0
data.json

@@ -0,0 +1 @@
+{"Sha1Sums": "", "Packages": "", "Sources": "", "next_adv": 0, "next_fsa": 0}

+ 1 - 0
debian-security-advisory.py → debian_advisory.py

@@ -12,6 +12,7 @@ import datetime
 from html.parser import HTMLParser
 from bs4 import BeautifulSoup
 from bs4 import NavigableString
+from pymongo import MongoClient
 import urllib.request
 import logging, sys