apt-sec.py~ 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. #!/usr/bin/python3
  2. ## New implementation of TrustMiner using python and mongodb
  3. ## Nikos
  4. import sys
  5. from pymongo import MongoClient
  6. #mongodb assumes database at default path
  7. import logging, sys
  8. import configparser
  9. import json
  10. import urllib.request
  11. import datetime
  12. import debian_advisory
  13. logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
  14. #load config file as library
  15. config = configparser.ConfigParser()
  16. config.read('config_test')
  17. if config.sections == []:
  18. print('configuration file not found\n')
  19. sys.exit(1)
  20. #global variables
  21. secperday = 60*60*24
  22. now = datetime.datetime.now()
  23. verbosity = 1
  24. ###############################################################################
  25. ## logging
  26. # 1 fatal errors
  27. # 2 errors
  28. # 3 note
  29. # 4 trace
  30. # 5 debug
  31. def msg(lvl,msg):
  32. if lvl <= int(config['LOG']['loglevel']):
  33. print(msg)
  34. def debug(msg):
  35. msg(5, msg)
  36. # Need to see if this is necessary
  37. ## load state, different from DBs in that we always need it
  38. def load_state():
  39. cache = config['DIR']['cache_dir'] + 'state'
  40. err = 0
  41. state = dict()
  42. try:
  43. with open(cache) as json_data:
  44. state = json.load(json_data)
  45. except FileNotFoundError:
  46. # Load default state - start from the beginning
  47. state['next_adv'] = 0
  48. state['next_fsa'] = 0
  49. state['Packages'] = ''
  50. state['Sources'] = ''
  51. state['Sha1Sums'] = ''
  52. err += 1
  53. return (state, err)
  54. ###############################################################################
  55. ## save state, different from DBs in that we always need it
  56. def save_state(state):
  57. cache = config['DIR']['cache_dir'] + 'state'
  58. try:
  59. with open(cache, 'w') as fp:
  60. json.dump(state, fp)
  61. except IOError:
  62. print('write cache failed!! Fatal error')
  63. sys.exit(1)
  64. ###############################################################################
  65. ## load sha lists :TODO later
  66. def load_sha1lists():
  67. cache = config['DIR']['cache_dir'] + 'state'
  68. ###############################################################################
  69. ## save sha lists :TODO later
  70. def save_sha1lists():
  71. pass
  72. ###############################################################################
  73. ## load from files :TODO later
  74. def load_DBs():
  75. pass
  76. ###############################################################################
  77. ## save to files :TODO later
  78. def save_DBs():
  79. pass
  80. ###############################################################################
  81. ## Fetch current Packages, Sources and sha1sums files
  82. ## These are needed to find CVE stats by sha1sums/pkg-names
  83. ## Only Sha1Sums is custom generated, others are from Debian.
  84. ## FIXME: Server might do on-the-fly gzip (but should not for bzip2)
  85. ## Return: 1 on success, to signal that new parsing is needed.
  86. def fetchMeta(filename):
  87. urlbase = config['URL']['pkg_base_url']
  88. mydir = config['DIR']['cache_dir']
  89. bzFile = filename + '.bz2'
  90. url = urlbase + bzFile
  91. logging.info('Checking meta file from ' + url + '\n')
  92. # Download file
  93. urllib.request.urlretrieve(url, mydir + bzfile)
  94. # TODO catch exceptions like file not found
  95. # TODO check if file has changed, if it is new unpack
  96. ###############################################################################
  97. # Sources and Packages are not completely consistent, esp for debian-multimedia
  98. # He we store manual mappings for these..
  99. def addOrphanPkgs(pkg2src):
  100. pkg2src['liblame-dev'] = "lame";
  101. pkg2src['lame-extras'] = "lame";
  102. pkg2src['moonlight'] = "moon";
  103. pkg2src['libmoon0'] = "moon";
  104. pkg2src['xmms-mp4'] = "xmms2";
  105. pkg2src['xmms-mp4'] = "xmms2";
  106. pkg2src['lazarus-src-0.9.30'] = "lazarus";
  107. pkg2src['lazarus-ide-0.9.30'] = "lazarus";
  108. pkg2src['lcl-qt4-0.9.30'] = "lazarus";
  109. pkg2src['lazarus-ide-qt4-0.9.30'] = "lazarus";
  110. pkg2src['lcl-gtk2-0.9.30'] = "lazarus";
  111. pkg2src['lazarus-ide-gtk2-0.9.30'] = "lazarus";
  112. pkg2src['lcl-units-0.9.30'] = "lazarus";
  113. pkg2src['lazarus-0.9.30'] = "lazarus";
  114. pkg2src['lazarus-doc-0.9.30'] = "lazarus";
  115. pkg2src['lcl-0.9.30'] = "lazarus";
  116. pkg2src['lcl-utils-0.9.30'] = "lazarus";
  117. pkg2src['lcl-nogui-0.9.30'] = "lazarus";
  118. pkg2src['libx264-65'] = "x264";
  119. pkg2src['libx264-114'] = "x264";
  120. pkg2src['libx264-60'] = "x264";
  121. # pkg2src['libmlt3']
  122. # pkg2src['libgmerlin-avdec0']
  123. # pkg2src['libxul-dev']
  124. # pkg2src['libmyth-0.23.1-0']
  125. # pkg2src['libmpeg3hv']
  126. # pkg2src['libquicktimehv']
  127. # pkg2src['libxul0d']
  128. # pkg2src['acroread-fonts-kor']
  129. ###############################################################################
  130. ## Parse dpkg Packages file, create map deb-name->pkg-name
  131. def parsePackages(pkgfile):
  132. mydir = cache = config['DIR']['cache_dir']
  133. deb2pkg = dict()
  134. pkg2virt = dict()
  135. virt2pkg = ()
  136. logging.info('Parsing Packages file...\n')
  137. pkgfile = mydir + pkgfile
  138. #TODO open and parse pkg file
  139. ###############################################################################
  140. ## Parse dpkg Sources file, create map pkg-name->src-name
  141. def parseSources(srcfile)
  142. mydir = cache = config['DIR']['cache_dir']
  143. checklinecont = 0
  144. pkg2src = dict()
  145. logging.info('Parsing Sources file...\n')
  146. srcfile = mydir + srcfile
  147. #TODO open and parse sources file
  148. ###############################################################################
  149. def getSHA1(myhash, collection):
  150. return collection.find({"hash": myhash})
  151. ###############################################################################
  152. def addSHA1(myhash, deb, src)
  153. dic = getSHA1(myhash)
  154. thash = dic["hash"]
  155. tdeb = dic["deb"]
  156. tsrc = dic["src"]
  157. #TODO insert SHA to database
  158. (state, err) = load_state()
  159. print(state)
  160. save_state(state)
  161. #client = MongoClient()
  162. #cve_db = client.cvedb
  163. #collection = db.cves
  164. #testcvss = collection.find_one({"cvss": 9.3})
  165. #print(testcvss)