|
@@ -26,6 +26,8 @@ import psycopg2
|
|
|
import powerlaw as pl
|
|
|
import DLAmine as dla
|
|
|
import pickle
|
|
|
+import paper_plots as carlosplt
|
|
|
+from matplotlib2tikz import save as tikz_save
|
|
|
|
|
|
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
|
|
|
## Increase the recursion limit by much to allow bs to parse large files
|
|
@@ -824,25 +826,26 @@ def plot_all(src2month):
|
|
|
## Sum of vulnerabilities by package
|
|
|
src2sum = dict()
|
|
|
src2year = dict()
|
|
|
- year_sum = [0] * 16
|
|
|
- year_num = [0] * 16
|
|
|
+ years = 16 # 2001 - 2000 + years
|
|
|
+ year_sum = [0] * years
|
|
|
+ year_num = [0] * years
|
|
|
for pkg in src2month:
|
|
|
- for j in range(16):
|
|
|
+ for j in range(years):
|
|
|
temp = sum(src2month[pkg][12*(1+j):12*(2+j)])
|
|
|
if (temp>0):
|
|
|
year_num[j] += 1
|
|
|
year_sum[j] += temp
|
|
|
- total = sum(src2month[pkg][12:-12])
|
|
|
- last_year = sum(src2month[pkg][-24:-12])
|
|
|
+ total = sum(src2month[pkg][12:])
|
|
|
+ last_year = sum(src2month[pkg][-12:])
|
|
|
print(pkg + '; ' + str(last_year))
|
|
|
- if (total>0):
|
|
|
+ if (total>1):
|
|
|
src2sum[pkg] = total
|
|
|
|
|
|
values = sorted(src2sum.values(),reverse=True)
|
|
|
keys = list(sorted(src2sum, key=src2sum.__getitem__, reverse=True))
|
|
|
i = 0
|
|
|
few_keys = []
|
|
|
- print(keys)
|
|
|
+ #print(keys)
|
|
|
for k in keys:
|
|
|
if (i==0):
|
|
|
few_keys.append(k)
|
|
@@ -851,105 +854,127 @@ def plot_all(src2month):
|
|
|
i = 0
|
|
|
|
|
|
print('package number =' + str(len(values)) + '... ' + str(len(keys)))
|
|
|
- plt.style.use('ggplot')
|
|
|
- plt.plot([1000] + values)
|
|
|
- #plt.xticks(np.arange(1,len(src2sum),10.0)+1,few_keys, rotation="vertical")
|
|
|
+ carlosplt.pre_paper_plot(True)
|
|
|
+ #plt.style.use('ggplot')
|
|
|
+
|
|
|
+ plt.figure(figsize=(10,5))
|
|
|
+ plt.plot([1000] + values, color='darkblue', lw = 2)
|
|
|
+ plt.xticks(np.arange(1,len(src2sum),10.0)+1,few_keys, rotation="vertical")
|
|
|
plt.ylabel('Vulnerabilities')
|
|
|
plt.yscale('log')
|
|
|
- plt.xscale('log')
|
|
|
+ #plt.xscale('log')
|
|
|
plt.tight_layout()
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
+ tikz_save('line.tex', figureheight='\\figureheight', figurewidth='\\figurewidth')
|
|
|
plt.show()
|
|
|
|
|
|
print('Yearly vulnerabilites in total' + str(year_sum))
|
|
|
|
|
|
## Number of affected packages
|
|
|
n = len(year_sum)
|
|
|
- yearsx = ['\'01', '\'02', '\'03', '\'04', '\'05', '\'06', '\'07', '\'08', '\'09', '\'10', '\'11' , '\'12' ,'\'13', '\'14', '\'15', '\'16']
|
|
|
- x = range(16)
|
|
|
+ yearsx = []
|
|
|
+ for i in range(1,years+1):
|
|
|
+ yearsx.append('\''+str(i).zfill(2))
|
|
|
+ x = range(years)
|
|
|
width = 1/2
|
|
|
- plt.bar(x, year_num, width, color='orange')
|
|
|
+ plt.bar(x, year_num, width, color='darkblue', edgecolor='black')
|
|
|
plt.xticks(np.arange(0,n),yearsx)
|
|
|
plt.ylabel('Number of affected packages')
|
|
|
plt.xlabel('Year')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
|
|
|
## Average number of vulnerabilities per package per year
|
|
|
- average_per_year = [0] * 16
|
|
|
- for j in range(16):
|
|
|
+ average_per_year = [0] * years
|
|
|
+ for j in range(years):
|
|
|
average_per_year[j] = year_sum[j]/float(year_num[j])
|
|
|
|
|
|
|
|
|
|
|
|
print(average_per_year)
|
|
|
+ x_values = list(range(1,years+1))
|
|
|
+ print(x_values)
|
|
|
+ slope = np.polyfit(x_values,average_per_year,1)
|
|
|
+
|
|
|
+ #slope = np.polyval(slope,x_values)
|
|
|
+ print('Slope: ' + str(slope))
|
|
|
|
|
|
|
|
|
n = len(year_sum)
|
|
|
- yearsx = ['\'01', '\'02', '\'03', '\'04', '\'05', '\'06', '\'07', '\'08', '\'09', '\'10', '\'11' , '\'12' ,'\'13', '\'14', '\'15', '\'16']
|
|
|
- x = range(16)
|
|
|
+ x = range(years)
|
|
|
width = 1/2
|
|
|
#plt.bar(x, year_sum, width)
|
|
|
- plt.bar(x, average_per_year, width, color='blue')
|
|
|
+ plt.bar(x, average_per_year, width, color='darkblue', edgecolor='black')
|
|
|
plt.xticks(np.arange(0,n),yearsx)
|
|
|
plt.ylabel('Average vulnerabilities per package')
|
|
|
plt.xlabel('Year')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
## Work on selected packages (php7.0, openjdk8, wireshark, chromium-browser, icedove, linux)
|
|
|
|
|
|
src2quarter = dict()
|
|
|
- quarter_num = 17*4
|
|
|
- for pkg in src2month:
|
|
|
- #if (pkg == 'php7.0' or pkg == 'openjdk-8' or pkg=='wireshark' or pkg=='chromium-browser' or pkg=='icedove' or pkg=='linux'):
|
|
|
- if (pkg == 'php5'):
|
|
|
- quarter_sum = [0] * quarter_num
|
|
|
- for j in range(quarter_num):
|
|
|
- temp = sum(src2month[pkg][12+3*j:12+3*(j+1)])
|
|
|
- quarter_sum[j] = temp
|
|
|
- src2quarter[pkg] = quarter_sum
|
|
|
+
|
|
|
+ quarter_num = years*4
|
|
|
+
|
|
|
+ pkg = 'php5'
|
|
|
+ quarter_sum = [0] * quarter_num
|
|
|
+ for j in range(quarter_num):
|
|
|
+ temp = sum(src2month[pkg][12+3*j:12+3*(j+1)])
|
|
|
+ quarter_sum[j] = temp
|
|
|
+ src2quarter[pkg] = quarter_sum
|
|
|
|
|
|
for pkg in src2quarter:
|
|
|
n = len(src2quarter[pkg])
|
|
|
- quartersx = ['1/\'01', '2/\'01', '3/\'01', '4/\'01', '1/\'02', '2/\'02', '3/\'02', '4/\'02', '1/\'03', '2/\'03', '3/\'03', '4/\'03', '1/\'04', '2/\'04', '3/\'04', '4/\'04', '1/\'05', '2/\'05', '3/\'05', '4/\'05', '1/\'06', '2/\'06', '3/\'06', '4/\'06', '1/\'07', '2/\'07', '3/\'07', '4/\'07', '1/\'08', '2/\'08', '3/\'08', '4/\'08', '1/\'09', '2/\'09', '3/\'09', '4/\'09', '1/\'10', '2/\'10', '3/\'10', '4/\'10', '1/\'11', '2/\'11', '3/\'11', '4/\'11', '1/\'12', '2/\'12', '3/\'12', '4/\'12', '1/\'13', '2/\'13', '3/\'13', '4/\'13', '1/\'14', '2/\'14', '3/\'14', '4/\'14', '1/\'15', '2/\'15', '3/\'15', '4/\'15', '1/\'16', '2/\'16', '3/\'16', '4/\'16', '1/\'17', '2/\'17', '3/\'17', '4/\'17']
|
|
|
- x = range(quarter_num)
|
|
|
- width = 1/2
|
|
|
- ## Plot different colors for php
|
|
|
- before = src2quarter[pkg][:-8] + ([0] * 8)
|
|
|
- after = ([0] * 60) + src2quarter[pkg][-8:]
|
|
|
- print(len(src2quarter[pkg]))
|
|
|
-
|
|
|
- bar1 = plt.bar(x[:-26], before[24:-2], width, color='blue', label='before php7')
|
|
|
- bar2 = plt.bar(x[:-26], after[24:-2], width, color='red', label='after php7')
|
|
|
- plt.legend(handles=[bar1, bar2])
|
|
|
-
|
|
|
- print('PHP Sum before: ' + str(sum(before)))
|
|
|
- print('PHP Sum after: ' + str(sum(after)))
|
|
|
-
|
|
|
- #plt.bar(x, src2quarter[pkg], width, color='red')
|
|
|
- plt.xticks(np.arange(0,n-26),quartersx[24:-2], rotation="vertical")
|
|
|
- plt.ylabel('Vulnerabilities per quarter of package ' + pkg)
|
|
|
- plt.xlabel('Quarter')
|
|
|
- plt.show()
|
|
|
+ quartersx = []
|
|
|
+ for i in range(1,years+1):
|
|
|
+ for j in range(1,5):
|
|
|
+ if j==1:
|
|
|
+ quartersx.append('Q' + str(j)+'\''+str(i).zfill(2))
|
|
|
+ else:
|
|
|
+ quartersx.append(' ')
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ x = range(quarter_num)
|
|
|
+ width = 1/2
|
|
|
+ ## Plot different colors for php
|
|
|
+ before = src2quarter[pkg][:-8] + ([0] * 8)
|
|
|
+ after = ([0] * (len(before)-8)) + src2quarter[pkg][-8:]
|
|
|
+ print(len(src2quarter[pkg]))
|
|
|
+#
|
|
|
+ bar1 = plt.bar(x[:-26], before[24:-2], width, color='darkblue', label='before php7', edgecolor='black')
|
|
|
+ bar2 = plt.bar(x[:-26], after[24:-2], width, color='darkred', label='after php7', edgecolor='black')
|
|
|
+ plt.legend(handles=[bar1, bar2])
|
|
|
+#
|
|
|
+ print('PHP Sum before: ' + str(sum(before)))
|
|
|
+ print('PHP Sum after: ' + str(sum(after)))
|
|
|
+
|
|
|
+ plt.xticks(np.arange(0,n-26),quartersx[24:-2], rotation="vertical")
|
|
|
+ plt.ylabel('Vulnerabilities per quarter of package ' + pkg)
|
|
|
+ plt.xlabel('Quarter')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
+ plt.show()
|
|
|
|
|
|
- ## Plot for openjdk-7
|
|
|
+# ## Plot for openjdk-7
|
|
|
pkg = 'openjdk-7'
|
|
|
quarter_sum = [0] * quarter_num
|
|
|
for j in range(quarter_num):
|
|
|
temp = sum(src2month[pkg][12+3*j:12+3*(j+1)])
|
|
|
quarter_sum[j] = temp
|
|
|
- src2quarter[pkg] = quarter_sum
|
|
|
+ src2quarter[pkg] = quarter_sum
|
|
|
|
|
|
n = len(src2quarter[pkg])
|
|
|
- quartersx = ['1/\'01', '2/\'01', '3/\'01', '4/\'01', '1/\'02', '2/\'02', '3/\'02', '4/\'02', '1/\'03', '2/\'03', '3/\'03', '4/\'03', '1/\'04', '2/\'04', '3/\'04', '4/\'04', '1/\'05', '2/\'05', '3/\'05', '4/\'05', '1/\'06', '2/\'06', '3/\'06', '4/\'06', '1/\'07', '2/\'07', '3/\'07', '4/\'07', '1/\'08', '2/\'08', '3/\'08', '4/\'08', '1/\'09', '2/\'09', '3/\'09', '4/\'09', '1/\'10', '2/\'10', '3/\'10', '4/\'10', '1/\'11', '2/\'11', '3/\'11', '4/\'11', '1/\'12', '2/\'12', '3/\'12', '4/\'12', '1/\'13', '2/\'13', '3/\'13', '4/\'13', '1/\'14', '2/\'14', '3/\'14', '4/\'14', '1/\'15', '2/\'15', '3/\'15', '4/\'15', '1/\'16', '2/\'16', '3/\'16', '4/\'16', '1/\'17', '2/\'17', '3/\'17', '4/\'17']
|
|
|
x = range(quarter_num)
|
|
|
width = 1/2
|
|
|
- ## Plot different colors for php
|
|
|
+# ## Plot different colors for openjdk
|
|
|
before = src2quarter[pkg][:-10] + ([0] * 10)
|
|
|
- after = ([0] * 58) + src2quarter[pkg][-10:]
|
|
|
+ after = ([0] * (len(before)-10)) + src2quarter[pkg][-10:]
|
|
|
print(len(src2quarter[pkg]))
|
|
|
|
|
|
- bar1 = plt.bar(x[:-48], before[48:], width, color='blue', label='before openjdk-8')
|
|
|
- bar2 = plt.bar(x[:-48], after[48:], width, color='red', label='after openjdk-8')
|
|
|
+ bar1 = plt.bar(x[:-48], before[48:], width, color='darkblue', label='before openjdk-8', edgecolor='black')
|
|
|
+ bar2 = plt.bar(x[:-48], after[48:], width, color='darkred', label='after openjdk-8', edgecolor='black')
|
|
|
plt.legend(handles=[bar1, bar2])
|
|
|
|
|
|
print('OpenJDK Sum before: ' + str(sum(before)))
|
|
@@ -959,6 +984,7 @@ def plot_all(src2month):
|
|
|
plt.xticks(np.arange(0,n-48),quartersx[48:], rotation="vertical")
|
|
|
plt.ylabel('Vulnerabilities per quarter of package ' + pkg)
|
|
|
plt.xlabel('Quarter')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
|
|
@@ -966,14 +992,14 @@ def plot_all(src2month):
|
|
|
###############################################################################################
|
|
|
|
|
|
n = len(year_sum)
|
|
|
- yearsx = ['\'01', '\'02', '\'03', '\'04', '\'05', '\'06', '\'07', '\'08', '\'09', '\'10', '\'11' , '\'12' ,'\'13', '\'14', '\'15', '\'16']
|
|
|
- x = range(16)
|
|
|
+ x = range(years)
|
|
|
width = 1/2
|
|
|
- plt.bar(x, year_sum, width)
|
|
|
+ plt.bar(x, year_sum, width, color='darkblue', edgecolor='black')
|
|
|
#plt.bar(x, average_per_year, width)
|
|
|
plt.xticks(np.arange(0,n),yearsx)
|
|
|
plt.ylabel('Total vulnerabilities')
|
|
|
plt.xlabel('Year')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
sumall = sum(values)
|
|
@@ -1004,15 +1030,17 @@ def plot_all(src2month):
|
|
|
LTS_quarter.append(temp)
|
|
|
|
|
|
## Print all LTS
|
|
|
+ cut = 12*4+1
|
|
|
n = len(quarter_sum)
|
|
|
x = range(quarter_num)
|
|
|
width = 1/2
|
|
|
|
|
|
- plt.bar(x, LTS_quarter, width, color='red', label='regular support')
|
|
|
+ plt.bar(x, LTS_quarter, width, color='brown', label='regular support', edgecolor='black')
|
|
|
|
|
|
plt.xticks(np.arange(0,n),quartersx, rotation="vertical")
|
|
|
plt.ylabel('Vulnerabilities per quarter of Debian LTS')
|
|
|
plt.xlabel('Quarter')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
|
|
@@ -1025,23 +1053,25 @@ def plot_all(src2month):
|
|
|
|
|
|
print(quarter_sum_errors)
|
|
|
|
|
|
- n = len(quarter_sum)
|
|
|
- x = range(quarter_num)
|
|
|
+ cut = 12*4+1
|
|
|
+ n = len(quarter_sum) - cut
|
|
|
+ x = range(quarter_num-cut)
|
|
|
width = 1/2
|
|
|
|
|
|
print(len(LTS_quarter))
|
|
|
|
|
|
- bar1 = plt.bar(x, quarter_sum_regular, width, color='brown', label='regular support')
|
|
|
- bar12 = plt.bar(x, quarter_sum_errors, width, color='orange', label='regular support after release of next version - may contain noise')
|
|
|
- bar2 = plt.bar(x, LTS_quarter, width, color='blue', label ='long-term support')
|
|
|
+ bar1 = plt.bar(x, quarter_sum_regular[cut:], width, color='darkblue', label='regular', edgecolor='black')
|
|
|
+ bar12 = plt.bar(x, quarter_sum_errors[cut:], width, color='darkorange', label='regular*', edgecolor='black')
|
|
|
+ bar2 = plt.bar(x, LTS_quarter[cut:], width, color='darkred', label ='long-term', edgecolor='black')
|
|
|
|
|
|
#bar1 = plt.bar(x[:-48], before[48:], width, color='blue', label='regular support')
|
|
|
#bar2 = plt.bar(x[:-48], after[48:], width, color='red', label='long-term support')
|
|
|
plt.legend(handles=[bar1, bar12, bar2])
|
|
|
|
|
|
- plt.xticks(np.arange(0,n),quartersx, rotation="vertical")
|
|
|
+ plt.xticks(np.arange(0,n),quartersx[cut:], rotation="vertical")
|
|
|
plt.ylabel('Vulnerabilities per quarter of Debian Wheezy')
|
|
|
plt.xlabel('Quarter')
|
|
|
+ carlosplt.post_paper_plot(True,True,True)
|
|
|
plt.show()
|
|
|
|
|
|
|
|
@@ -1098,17 +1128,12 @@ def aptsec_help():
|
|
|
##
|
|
|
## Note: aptsec_status(), considers *reportedly installed* packages, while this
|
|
|
## one looks at *actually loaded* software that influenced the CPU since bootup.
|
|
|
-def aptsec_attest(sha1file):
|
|
|
- pass
|
|
|
-
|
|
|
-## Main Program starts here!!
|
|
|
-
|
|
|
try:
|
|
|
action = sys.argv[1]
|
|
|
except IndexError:
|
|
|
-# print('No argument given')
|
|
|
-# aptsec_help()
|
|
|
-# sys.exit(0)
|
|
|
+ print('No argument given')
|
|
|
+ aptsec_help()
|
|
|
+ sys.exit(0)
|
|
|
action = ''
|
|
|
|
|
|
|
|
@@ -1144,7 +1169,7 @@ if action == 'update':
|
|
|
# getdeps(src2dsa, src2deps)
|
|
|
save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps, src2sum)
|
|
|
save_state(state)
|
|
|
- lstm.predict(src2month, src2sloccount, src2pop, src2deps)
|
|
|
+# lstm.predict(src2month, src2sloccount, src2pop, src2deps)
|
|
|
# print(pkg_with_cvss['linux'])
|
|
|
|
|
|
low = []
|