added cve-search

CVE-search/cve-search

@@ -0,0 +1 @@
+Subproject commit 17e67c6d0d7ce3673af1a3771c4544a296995089

apt-sec.py

@@ -0,0 +1,4 @@
+#!/usr/bin/python3
+import logging, sys
+
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

common-vulnerability-entry.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+
+###############################################################################
+##
+## Functions for downloading and parsing Common Vulnerability DB data
+##
+###############################################################################
+
+import logging, sys
+import re
+import urllib.request
+import datetime
+
+logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+## Get details of given CVE entry from NIST DB
+def fetchCVE(cve_id, cve_base_url, cvss_base_url):
+    url = ''
+
+    logging.info('Fetching CVE: ' + cve_id + '\n')
+
+    #Make this subtitution for some reason
+    cve_id = re.sub('^CAN','CVE',cve_id)
+
+    ##
+    ## get CVE Scores from NVD
+    ##
+
+    url = cvss_base_url + cve_id
+    try:
+        req = urllib.request.urlopen(url)
+        charset = req.info().get_content_charset()
+        if charset is None:
+            charset = 'utf-8'
+        cvss = req.read().decode(charset)
+    except urllib.error.HTTPError as err:
+        print('Failed to download CVE: ' + url + '\n')
+        cvss = ''
+
+    # Check for error pages: referenced but unpublished CVEs :-/
+    if re.compile('.*is valid CVE format, but CVE was not found.*').match(cvss):
+        print(cve_id + ' does not exist in NIST DB\n')
+
+	##
+	## get CVE from MITRE
+	##
+        
+    logging.info('Fetching CVE: ' + cve_id + ' from MITRE\n')
+
+    url = cve_base_url + cve_id
+    try:
+        req = urllib.request.urlopen(url)
+        charset = req.info().get_content_charset()
+        if charset is None:
+            charset = 'utf-8'
+        cve = req.read().decode(charset)
+    except urllib.error.HTTPError as err:
+        print('Failed to download CVE: ' + url + ' from MITRE\n')
+        cve = ''
+        return cve
+
+    # Check for error pages: referenced but unpublished CVEs :-/
+    if re.compile('.*Could not find a CVE entry or candidate named.*').match(cve):
+        print(cve_id + 'does not exist in MITRE DB\n')
+        return ''
+
+    s = ''
+    ret = s.join((cvss,cve))
+    return ret
+###############################################################################
+
+## Get CVE severity rating and report date, and return
+## (date base-score impact-score exploit-score)
+def parseCVE(cve_id, cve):
+    #use worst case info as defaults
+    cve_date = datetime.datetime.now()
+    print(cve_date)
+    cve_base = 10
+    cve_impact = 10
+    cve_exploit = 10
+
+    if cve == '':
+        # No details means we assume worst-case (highest score, recent bug)
+        if re.compile('LOCAL-(.*)').match(cve_id):
+            
+
+
+
+
+#test1 = fetchCVE('CVE-2015-2750', 'http://cve.mitre.org/cgi-bin/cvename.cgi?name=', 'http://web.nvd.nist.gov/view/vuln/detail?vulnId=')
+#print(test1)
+parseCVE('a','b')

debian-security-advisory.py

@@ -11,6 +11,7 @@ import re
 import datetime
 from html.parser import HTMLParser
 from bs4 import BeautifulSoup
+from bs4 import NavigableString
 import urllib.request
 import logging, sys
 
@@ -18,9 +19,340 @@ logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
 
 #Testing global variables
 config = dict([('dsa_base_url','https://www.debian.org/security/')])
-state = dict([('next_adv',3700)])
+state = dict([('next_adv',3496)])
 dsatable = dict()
 
+
+# Track renamed packages here, easy but manual. We should look into ways
+# to automate this
+
+def unifySrcName(name):
+    	## TODO: it should map to the most recent version, not unversioned
+	## TODO: we can partially automate this..
+	## -> make all lower-case
+	## -> replace -X.Y version numbers by highest encounter(?)
+	## -> handle special cases like xfree86
+        # Have to go over this again!
+        # Maybe have a file with all these things??
+        newname = name
+        if re.compile("proftp-dfsg").match(name):
+            newname = "proftp" 
+        elif re.compile("mozilla-firefox").match(name):
+            newname = "iceweasel" 
+        elif re.compile("mozilla-thunderbird").match(name):
+            newname = "icedove" 
+        elif re.compile("squid3").match(name):
+            newname = "squid" 
+        elif re.compile("squid/squid3").match(name):
+            newname = "squid" 
+        elif re.compile("tk8.3").match(name):
+            newname = "tk8.4" 
+        elif re.compile("tk8.4").match(name):
+            newname = "tk8.4" 
+        elif re.compile("xpdf-i").match(name):
+            newname = "xpdf" 
+        elif re.compile("zope2.10/zope2.9").match(name):
+            newname = "zope2.7" 
+        elif re.compile("zope-cmfplone").match(name):
+            newname = "zope2.7" 
+        elif re.compile("zope-ldapuserfolder").match(name):
+            newname = "zope2.7" 
+        elif re.compile("librmagick-ruby").match(name):
+            newname = "ruby-rmagick" 
+        elif re.compile("libcompass-ruby").match(name):
+            newname = "ruby-compass" 
+        elif re.compile("bio-ruby").match(name):
+            newname = "ruby-bio" 
+        elif re.compile("request-tracker3.4").match(name):
+            newname = "request-tracker3.8" 
+        elif re.compile("request-tracker3.6").match(name):
+            newname = "request-tracker3.8" 
+        elif re.compile("perl-5.005").match(name):
+            newname = "perl" 
+        elif re.compile("otrs2").match(name):
+            newname = "otrs" 
+        elif re.compile("openldap2.3").match(name):
+            newname = "openldap" 
+        elif re.compile("openldap2").match(name):
+            newname = "openldap" 
+        elif re.compile("libreoffice").match(name):
+            newname = "openoffice.org" 
+        elif re.compile("nsd3").match(name):
+            newname = "nsd" 
+        elif re.compile("network-manager/network-manager-applet").match(name):
+            newname = "network-manager" 
+        elif re.compile("nagios3").match(name):
+            newname = "nagios" 
+        elif re.compile("nagios2").match(name):
+            newname = "nagios" 
+        elif re.compile("mysql-dfsg-4.1").match(name):
+            newname = "mysql" 
+        elif re.compile("mysql-dfsg-5.0").match(name):
+            newname = "mysql" 
+        elif re.compile("mysql-dfsg").match(name):
+            newname = "mysql" 
+        elif re.compile("linux-2.6.24").match(name):
+            newname = "linux-2.6" 
+        elif re.compile("linux-kernel-alpha").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("linux-kernel-i386").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("libmusicbrainz-2.0").match(name):
+            newname = "libmusicbrainz3" 
+        elif re.compile("libmusicbrainz-2.1").match(name):
+            newname = "libmusicbrainz3" 
+        elif re.compile("libgtop1").match(name):
+            newname = "libgtop2" 
+        elif re.compile("libgd1").match(name):
+            newname = "libgd2" 
+        elif re.compile("libast1").match(name):
+            newname = "libast" 
+        elif re.compile("libmozjs0d").match(name):
+            newname = "libast" 
+        elif re.compile("^kernel-source-2.2.*").match(name):
+            newname = "linux-2.2" 
+        elif re.compile("^kernel-patch-2.2.*").match(name):
+            newname = "linux-2.2" 
+        elif re.compile("kernel").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("^kernel-source-2.4.*").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("^kernel-image-2.2.*").match(name):
+            newname = "linux-2.2" 
+        elif re.compile("^kernel-image-2.4.*").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("^kernel-patch-2.4.*").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("kernel-patch-benh").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("kernel-patch-vserver").match(name):
+            newname = "linux-2.4" 
+        elif re.compile("^kernel-source-2.6.*").match(name):
+            newname = "linux-2.6" 
+        elif re.compile("gnutls11").match(name):
+            newname = "gnutls26" 
+        elif re.compile("gnutls13").match(name):
+            newname = "gnutls26" 
+        elif re.compile("gallery2").match(name):
+            newname = "gallery" 
+        elif re.compile("firebird2").match(name):
+            newname = "firebird2.5" 
+        elif re.compile("firebird2.0").match(name):
+            newname = "firebird2.5" 
+        elif re.compile("firebird2.1").match(name):
+            newname = "firebird2.5" 
+        elif re.compile("fltk1.1").match(name):
+            newname = "fltk1.3" 
+        elif re.compile("fox1.4").match(name):
+            newname = "fox1.6" 
+        elif re.compile("exim-tls").match(name):
+            newname = "exim" 
+        elif re.compile("exim4").match(name):
+            newname = "exim" 
+        elif re.compile("epic4").match(name):
+            newname = "epic" 
+        elif re.compile("drupal6").match(name):
+            newname = "drupal" 
+        elif re.compile("dhcp").match(name):
+            newname = "dhcp3" 
+        elif re.compile("cyrus-sasl").match(name):
+            newname = "cyrus-sasl2" 
+        elif re.compile("^cyrus-imapd.*").match(name):
+            newname = "cyrus-imapd" 
+        elif re.compile("^kolab-cyrus-imapd.*").match(name):
+            newname = "cyrus-imapd" 
+        elif re.compile("cfengine").match(name):
+            newname = "cfengine2" 
+        elif re.compile("bind").match(name):
+            newname = "bind9" 
+        elif re.compile("apache").match(name):
+            newname = "apache2" 
+        elif re.compile("horde2").match(name):
+            newname = "horde3" 
+        elif re.compile("mediawiki1.7").match(name):
+            newname = "mediawiki" 
+        elif re.compile("ffmpeg-debian").match(name):
+            newname = "ffmpeg" 
+        elif re.compile("xserver-xorg").match(name):
+            newname = "xorg-server" 
+        elif re.compile("xfree86-1").match(name):
+            newname = "xorg-server" 
+        elif re.compile("xfree86v3").match(name):
+            newname = "xorg-server" 
+        elif re.compile("xfree86").match(name):
+            newname = "xorg-server" 
+        elif re.compile("xfree86").match(name):
+            newname = "xorg-server" 
+        elif re.compile("xorg").match(name):
+            newname = "xorg-server" 
+        elif re.compile("typo3").match(name):
+            newname = "typo3-src" 
+        elif re.compile("lvm10").match(name):
+            newname = "lvm2" 
+        elif re.compile("cupsys").match(name):
+            newname = "cups" 
+        elif re.compile("ethereal").match(name):
+            newname = "wireshark" 
+        elif re.compile("libboost1.42").match(name):
+            newname = "libboost1.46" 
+        elif re.compile("cinelerra-cv").match(name):
+            newname = "cinelerra" 
+        elif re.compile("mplayer-dmo").match(name):
+            newname = "mplayer" 
+        elif re.compile("libcap").match(name):
+            newname = "libgda2" 
+        elif re.compile("xkb-data-legacy").match(name):
+            newname = "xkeyboard-config" 
+        elif re.compile("boost-defaults").match(name):
+            newname = "boost" 
+        elif re.compile("xen-3").match(name):
+            newname = "xen" 
+        elif re.compile("kde-icons-gorilla").match(name):
+            newname = "kde-icons-korilla" 
+        elif re.compile("kde4libs").match(name):
+            newname = "kdelibs" 
+        elif re.compile("libcgi-application-extra-plugin-bundle-perl").match(name):
+            newname = "libcgi-application-plugins-perl"
+        elif re.compile("^openssl\d$").match(name):
+            newname = "openssl" 
+        elif re.compile("^tomcat\d").match(name):
+            newname = "tomcat7" 
+        elif re.compile("^tomcat\d.\d$").match(name):
+            newname = "tomcat7" 
+        elif re.compile("^libgda\d").match(name):
+            newname = "libgda4" 
+        elif re.compile("^readline\d").match(name):
+            newname = "readline6" 
+        elif re.compile("^libwnck\d").match(name):
+            newname = "libwnck" 
+        elif re.compile("^xerces-c\d").match(name):
+            newname = "xerces-c" 
+        elif re.compile("^libticalcs\d").match(name):
+            newname = "libticals" 
+        elif re.compile("^libtifiles\d").match(name):
+            newname = "libtifiles" 
+        elif re.compile("^db\d.\d$").match(name):
+            newname = "db4.8" 
+        elif re.compile("^gcc-.*").match(name):
+            newname = "gcc" 
+        elif re.compile("^automake\d+.*").match(name):
+            newname = "automake" 
+        elif re.compile("^sun-java\d").match(name):
+            newname = "sun-java6" 
+        elif re.compile("^open-jdk\d").match(name):
+            newname = "open-jdk7" 
+        elif re.compile("^mbrola-es\d").match(name):
+            newname = "mbrola-es" 
+        elif re.compile("^mgltools-.*").match(name):
+            newname = "mgltools" 
+        elif re.compile("^coin\d$").match(name):
+            newname = "coin" 
+        elif re.compile("^adobereader-\.*").match(name):
+            newname = "adobereader" 
+        elif re.compile("^picon-\.*").match(name):
+            newname = "picon" 
+        elif re.compile("^nvidia-graphics-drivers\.*").match(name):
+            newname = "nvidia-graphics-drivers" 
+        elif re.compile("^boost\d\.\d\d").match(name):
+            newname = "boost" 
+        elif re.compile("^llvm-\d.\d").match(name):
+            newname = "llvm" 
+        elif re.compile("^octave\d.\d").match(name):
+            newname = "octave" 
+        elif re.compile("^libjibx\d.\d-java").match(name):
+            newname = "libjibx-java" 
+        elif re.compile("^emacs2\d").match(name):
+            newname = "emacs23" 
+        elif re.compile("^emacs2\d-non-dfsg").match(name):
+            newname = "emacs23" 
+        elif re.compile("^libupnp\d").match(name):
+            newname = "libupnp" 
+        elif re.compile("^python\d.\d").match(name):
+            newname = "python3.2" 
+        elif re.compile("^postgresql-\d.\d").match(name):
+            newname = "postgresql-9.0" 
+        elif re.compile("^ruby\d.\d").match(name):
+            newname = "ruby1.9" 
+        elif re.compile("^php\d").match(name):
+            newname = "php5" 
+        elif re.compile("^PHP\d").match(name):
+            newname = "php5" 
+            
+        return newname
+
+###############################################################################
+
+## Should this advisory be skipped?+
+def blacklistedDSA(dsa_id):
+    dsa_blacklist = ["DSA-1975", "DSA-2360"]
+    if dsa_id in dsa_blacklist:
+        return True
+    else:
+        return False
+###############################################################################
+
+## Static map to correct errors in DSAs
+## Return fixed list of CVE IDs or 0 to skip DSA
+## This code is still experimental
+def fixDSAquirks(dsa_id, dsa_state):
+    new_names = dsa_state[0]
+    new_date = dsa_state[1]
+    new_cves = dsa_state[2]
+
+    if dsa_id == "085":
+        new_cves = ["CVE-2001-1562", "LOCAL-03/04/05", "LOCAL-08/24/08"]
+    elif dsa_id == "745":
+        newcves = ["CVE-2005-1921", "CVE-2005-2106", "CVE-2005-1921"]
+    elif dsa_id == "1095":
+        new_cves = ["CVE-2006-0747", "CVE-2006-1861", "CVE-2006-2661"]
+    elif dsa_id == "1284":
+        new_cves = ["CVE-2007-1320", "CVE-2007-1321", "CVE-2007-1322", "CVE-2007-2893", "CVE-2007-1366"]
+    elif dsa_id == "1502":
+        new_cves = ["CVE-2007-2821", "CVE-2007-3238", "CVE-2008-0193", "CVE-2008-0194"]
+    elif dsa_id == "1706":
+        new_cves = ["CVE-2009-0135", "CVE-2009-0136"]
+    elif dsa_id == "1757":
+        new_cves = ["CVE-2007-2383", "CVE-2008-7220", "CVE-2009-1208"]
+    elif dsa_id == "1896":
+        new_cves = ["CVE-2009-3474", "CVE-2009-3475", "CVE-2009-3476"]
+    elif dsa_id == "1931":
+        new_cves = ["CVE-2009-0689", "CVE-2009-2463"]
+    elif dsa_id == "1989":
+        new_cves = ["CVE-2010-0789"]
+    elif dsa_id == "1941":
+        new_cves = ["CVE-2009-0755", "CVE-2009-3903", "CVE-2009-3904", "CVE-2009-3905", "CVE-2009-3606", "CVE-2009-3607", "CVE-2009-3608", "CVE-2009-3909", "CVE-2009-3938"]
+    elif dsa_id == "2004":
+        new_cves = ["CVE-2010-0787", "CVE-2010-0547"]
+    elif dsa_id == "2008":
+        new_cves = ["LOCAL-02/23/10", "LOCAL-02/23/10", "LOCAL-02/23/10", "LOCAL-02/23/10"]
+    elif dsa_id == "2043":
+        new_cves = ["CVE-2010-2062"]
+    elif dsa_id == "2044":
+        new_cves = ["CVE-2010-2062"]
+    elif dsa_id == "2056":
+        new_cves = ["CVE-2010-2155", "CVE-2009-4882"]
+    elif dsa_id == "2092":
+        new_cves = ["CVE-2010-1625", "CVE-2010-1448", "CVE-2009-4497"]
+    elif dsa_id == "2098":
+        new_cves = ["CVE-2010-3659", "CVE-2010-3660", "CVE-2010-3661", "CVE-2010-3662", "CVE-2010-3663", "CVE-2010-3664", "CVE-2010-3665", "CVE-2010-3666", "CVE-2010-3667", "CVE-2010-3668", "CVE-2010-3669", "CVE-2010-3670", "CVE-2010-3671", "CVE-2010-3672", "CVE-2010-3673", "CVE-2010-3674"]
+    elif dsa_id == "2103":
+        new_cves = ["CVE-2010-3076"]
+    elif dsa_id == "2218":
+        new_cves = ["CVE-2011-1684"]
+    elif dsa_id == "2229":
+        new_cves = ["CVE-2005-4494", "CVE-2006-0517", "CVE-2006-0518", "CVE-2006-0519", "CVE-2006-0625", "CVE-2006-0626", "CVE-2006-1295", "CVE-2006-1702", "CVE-2007-4525", "CVE-2008-5812", "CVE-2008-5813", "CVE-2009-3041"]
+    elif dsa_id == "2261":
+        new_cves = ["CVE-2009-4078", "CVE-2009-4079", "CVE-2009-4059", "LOCAL-12/30/10", "LOCAL-12/30/10"]
+    elif dsa_id == "2262":
+        new_cves = ["LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-03/01/11", "LOCAL-05/18/11", "LOCAL-05/18/11"]
+    elif dsa_id == "2286":
+        new_names = ["phpmyadmin"]
+    elif dsa_id == "2289":
+        new_cves = ["LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11", "LOCAL-07/27/11"]
+
+    return (new_names, new_date, new_cves)
+###############################################################################
+
 ## Fetch DSA from debian archive. Can't use tracker since dates are missing.
 ## DSA started counting in November 2000. We'll simply bruteforce which DSA
 ## was in which year and start in 2000 til current.
@@ -97,8 +429,9 @@ def checkDSAs(state, config):
 #         logging.debug('Cache was deleted, starting at DSA ' + str(next_dsa) + '\n')
 #         next_dsa = config['first_dsa']
 
-#    if  blacklistedDSA('DSA-' + str(next_dsa)):
-#        next_dsa += 1
+    next_dsa2string = '%03d' % next_dsa
+    if  blacklistedDSA('DSA-' + next_dsa2string):
+        next_dsa += 1
 
     dsa = fetchDSA(next_dsa, config['dsa_base_url'])
     
@@ -111,8 +444,8 @@ def checkDSAs(state, config):
             raise NameError('html file format unexpected')
         dsatable[next_dsa] = str(dsa)
         next_dsa += 1
-#        if  blacklistedDSA('DSA-' + str(next_dsa)):
-#            next_dsa += 1
+        if  blacklistedDSA('DSA-' + str(next_dsa)):
+            next_dsa += 1
         dsa = fetchDSA(next_dsa, config['dsa_base_url'])
 
     state['next_dsa'] = next_dsa
@@ -120,42 +453,49 @@ def checkDSAs(state, config):
 
 ###############################################################################
 
-
 ## Parse DSA html data and return array
 ## (src-pkg-name date (CVE-id)*)
 
 def parseDSAhtml(dsa):
     
+    dsa_date = []
+    dsa_names = []
+    dsa_CVEs = []
     # Date Reported -> dsa_date
     soup = BeautifulSoup(dsa, 'html.parser')
     tmp = soup.find("dt",string=re.compile(".*Date Repo.*:"))
     tmp = str(tmp.find_next().contents[0])
     dsa_date = tmp.split()
     if dsa_date == []:
-        print('Unable to extract date. Returning...')
-        raise NameError('file format problem')
+        print('Unable to extract date. Raising exception...')
+        raise NameError('DSA parsing problem')
 
     # Affected Packages -> dsa_names
     #print(dsa)
     tmp = soup.find("dt",string=re.compile("Affected Packages:"))
     tmp = tmp.find_next().contents
-
+    #Need to check with multiple vulnerable packages
     for i in tmp:
-        print(i)
-    
-#    m = re.search('<dt>Affected\ Packages:<\/dt>(.*)<\/dd>.*<dt>Vulnerable:',dsa)
-#    if m:
-#        print(len(m.group))
-#        tmpstring = m.group(1)
-#    else:
-#        print('Unable to extract affected packages. Returning...')
-#        raise NameError('html file format unexpected')
-
+        if (not isinstance(i, NavigableString)) and i.has_attr('href'):
+            #greedy 'and' operation assumed
+            unified = unifySrcName(i.string)
+            dsa_names.append(unified)
+            pass
+    if dsa_names == []:
+        print('Unable to find src package in DSA. Raising exception...')
+        raise NameError('DSA parsing problem')
 
+    # Security database references (CVEs) -> dsa_CVEs
+    tmp = soup.find("dt", string=re.compile("Security database references:"))
+    tmp = tmp.find_next().descendants
+    for i in tmp:
+         if (not isinstance(i, NavigableString)) and i.has_attr('href'):
+            dsa_CVEs.append(i.string)
 
+    return (dsa_names, dsa_date, dsa_CVEs)
 
 #dsa = fetchDSA(3200,'https://www.debian.org/security/')
-dsatable = checkDSAs(state,config)
+#dsatable = checkDSAs(state,config)
 #print(dsatable[3701])
-parseDSAhtml(dsatable[3701])
+#parseDSAhtml(dsatable[3498])
 #checkDSAs(state,config)