TrustMiner2/apt-sec.py~

#!/usr/bin/python3
## New implementation of TrustMiner using python and mongodb
## Nikos

import sys
from pymongo import MongoClient
#mongodb assumes database at default path
import logging, sys
import configparser
import json
import urllib.request
import datetime
import debian_advisory

logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

#load config file as library
config = configparser.ConfigParser()
config.read('config_test')
if config.sections == []:
    print('configuration file not found\n')
    sys.exit(1)

#global variables
secperday = 60*60*24
now = datetime.datetime.now()
verbosity = 1

###############################################################################

## logging
# 1 fatal errors
# 2 errors
# 3 note
# 4 trace
# 5 debug

def msg(lvl,msg):
    if lvl <= int(config['LOG']['loglevel']):
        print(msg)

def debug(msg):
    msg(5, msg)
# Need to see if this is necessary

## load state, different from DBs in that we always need it
def load_state():
    cache = config['DIR']['cache_dir'] + 'state'
    err = 0
    state = dict()

    try:
        with open(cache) as json_data:
            state = json.load(json_data)
    except FileNotFoundError:
        # Load default state - start from the beginning
        state['next_adv'] = 0
        state['next_fsa'] = 0
        state['Packages'] = ''
        state['Sources'] = ''
        state['Sha1Sums'] = ''
        err += 1

    return (state, err)

###############################################################################
## save state, different from DBs in that we always need it
def save_state(state):
    cache = config['DIR']['cache_dir'] + 'state'
    
    try:
        with open(cache, 'w') as fp:
            json.dump(state, fp)
    except IOError:
        print('write cache failed!! Fatal error')
        sys.exit(1)

###############################################################################
## load sha lists :TODO later
def load_sha1lists():
    cache = config['DIR']['cache_dir'] + 'state'

###############################################################################
## save sha lists :TODO later
def save_sha1lists():
    pass

###############################################################################
## load from files :TODO later
def load_DBs():
    pass

###############################################################################
## save to files :TODO later
def save_DBs():
    pass

###############################################################################
## Fetch current Packages, Sources and sha1sums files
## These are needed to find CVE stats by sha1sums/pkg-names
## Only Sha1Sums is custom generated, others are from Debian.
## FIXME: Server might do on-the-fly gzip (but should not for bzip2)
## Return: 1 on success, to signal that new parsing is needed.
def fetchMeta(filename):
    urlbase = config['URL']['pkg_base_url']
    mydir = config['DIR']['cache_dir']
    bzFile = filename + '.bz2'
    url = urlbase + bzFile

    logging.info('Checking meta file from ' + url + '\n')

    # Download file
    urllib.request.urlretrieve(url, mydir + bzfile)
    # TODO catch exceptions like file not found
    # TODO check if file has changed, if it is new unpack

###############################################################################
# Sources and Packages are not completely consistent, esp for debian-multimedia
# He we store manual mappings for these..
def addOrphanPkgs(pkg2src):
    pkg2src['liblame-dev'] = "lame";
    pkg2src['lame-extras'] = "lame";
    pkg2src['moonlight'] = "moon";
    pkg2src['libmoon0'] = "moon";
    pkg2src['xmms-mp4'] = "xmms2";
    pkg2src['xmms-mp4'] = "xmms2";
    pkg2src['lazarus-src-0.9.30'] = "lazarus";
    pkg2src['lazarus-ide-0.9.30'] = "lazarus";
    pkg2src['lcl-qt4-0.9.30'] = "lazarus";
    pkg2src['lazarus-ide-qt4-0.9.30'] = "lazarus";
    pkg2src['lcl-gtk2-0.9.30'] = "lazarus";
    pkg2src['lazarus-ide-gtk2-0.9.30'] = "lazarus";
    pkg2src['lcl-units-0.9.30'] = "lazarus";
    pkg2src['lazarus-0.9.30'] = "lazarus";
    pkg2src['lazarus-doc-0.9.30'] = "lazarus";
    pkg2src['lcl-0.9.30'] = "lazarus";
    pkg2src['lcl-utils-0.9.30'] = "lazarus";
    pkg2src['lcl-nogui-0.9.30'] = "lazarus";
    pkg2src['libx264-65'] = "x264";
    pkg2src['libx264-114'] = "x264";
    pkg2src['libx264-60'] = "x264";
#  pkg2src['libmlt3']
#  pkg2src['libgmerlin-avdec0']
#  pkg2src['libxul-dev']
#  pkg2src['libmyth-0.23.1-0']
#  pkg2src['libmpeg3hv']
#  pkg2src['libquicktimehv']
#  pkg2src['libxul0d']
#  pkg2src['acroread-fonts-kor']

###############################################################################
## Parse dpkg Packages file, create map deb-name->pkg-name
def parsePackages(pkgfile):
    mydir = cache = config['DIR']['cache_dir']
    deb2pkg = dict()
    pkg2virt = dict()
    virt2pkg = ()

    logging.info('Parsing Packages file...\n')
    pkgfile = mydir + pkgfile

    #TODO open and parse pkg file

###############################################################################
## Parse dpkg Sources file, create map pkg-name->src-name
def parseSources(srcfile)
    mydir = cache = config['DIR']['cache_dir']
    checklinecont = 0
    pkg2src = dict()

    logging.info('Parsing Sources file...\n')
    srcfile = mydir + srcfile
    
    #TODO open and parse sources file


###############################################################################
def getSHA1(myhash, collection):
    return collection.find({"hash": myhash})


###############################################################################
def addSHA1(myhash, deb, src)
    dic = getSHA1(myhash)
    thash = dic["hash"]
    tdeb = dic["deb"]
    tsrc = dic["src"]

    #TODO insert SHA to database






(state, err) = load_state()
print(state)
save_state(state)

#client = MongoClient()

#cve_db = client.cvedb
#collection = db.cves
#testcvss = collection.find_one({"cvss": 9.3})
#print(testcvss)