a litle housekeeping

apt-sec.py

@@ -890,12 +890,12 @@ if action == 'update':
     aptsec_update(state,config, dsatable, client, src2dsa, dsa2cve, src2month, cvetable, pkg_with_cvss)
 #    save_sha1lists()
 #    getslocs(src2dsa, src2sloccount)
-    getpop(src2dsa, src2pop)
-    getdeps(src2dsa, src2deps)
+#   getpop(src2dsa, src2pop)
+#    getdeps(src2dsa, src2deps)
     save_DBs(dsatable, src2dsa, dsa2cve, cvetable, src2month, src2sloccount, src2pop, src2deps)
     save_state(state)
 #    ml.predict(src2month)
-#    lstm.predict(src2month)
+    lstm.predict(src2month)
 #    print(pkg_with_cvss['linux'])
     
     low = []

lstm_reg.py

@@ -15,7 +15,7 @@ numpy.random.seed(7)
 def create_dataset(dataset, look_back=1):
 	dataX, dataY = [], []
 	for i in range(len(dataset)-look_back-1):
-		a = dataset[i:(i+look_back-6)]
+		a = dataset[i:(i+look_back)]
 		dataX.append(a)
 		dataY.append(dataset[i + look_back])
 	return numpy.array(dataX), numpy.array(dataY)
@@ -23,33 +23,31 @@ def create_dataset(dataset, look_back=1):
 def predict(src2month):
 
     pkg_num = len(src2month)
-    training_num = len(src2month['linux'])-12
+    training_num = len(src2month['linux'])
 
-    look_back = 24
+    look_back = 4
     # create the LSTM network
     model = Sequential()
-    model.add(LSTM(128, input_dim=look_back-6, activation ='relu', dropout_W =0.6, dropout_U =0.6))
-    model.add(Dense(12, init='uniform', activation='relu'))
-    model.add(Dense(8, init='uniform', activation='relu'))
-    model.add(Dense(1, init='uniform', activation='sigmoid'))
+    model.add(LSTM(64, input_dim=look_back, activation ='relu', dropout_W =0.1, dropout_U =0.1))
+#    model.add(Dense(12, init='uniform', activation='relu'))
+#    model.add(Dense(8, init='uniform', activation='relu'))
+#    model.add(Dense(1, init='uniform', activation='sigmoid'))
 #    model.add(LSTM(4, input_dim=look_back-6, dropout_W = 0.2, dropout_U = 0.1))
-#    model.add(Dense(1))
+    model.add(Dense(1))
     model.compile(loss='mean_squared_error', optimizer='adam')
     
+    scaler = MinMaxScaler(feature_range=(0, 1))
 
     flag = True
 ###################################################################################################    
-    for pkg_name in ['icedove', 'linux', 'mysql', 'xulrunner', 'wireshark', 'wordpress', 'iceape', 'xen', 'asterisk', 'tomcat7', 'phpmyadmin', 'asterisk', 'mariadb-10.0', 'libxml2', 'apache2', 'cups', 'samba', 'freetype', 'tiff', 'clamav', 'bind9', 'squid', 'openssl', 'moodle', 'cacti', 'krb5', 'ffmpeg', 'mantis', 'xpdf', 'imagemagick', 'typo3-src', 'firefox', 'chromium-browser']:
- 
+    for pkg_name in ['icedove', 'linux', 'mysql', 'xulrunner', 'wireshark', 'firefox', 'openjdk', 'php5', 'iceape', 'wordpress', 'xen', 'openssl', 'chromium-browser']:
 #    for pkg_name in ['chromium-browser']:
         pkg_num = len(src2month)
-        training_num = len(src2month['linux'])-12
         dataset = src2month[pkg_name]
         dataset = pandas.rolling_mean(dataset, window=12)
         dataset = dataset[12:]
 
         # normalize the dataset
-        scaler = MinMaxScaler(feature_range=(0, 1))
         dataset = scaler.fit_transform(dataset)
 
         train_size = int(len(dataset) * 0.80)
@@ -60,32 +58,14 @@ def predict(src2month):
         # reshape into X=t and Y=t+1
         trainX, trainY = create_dataset(train, look_back)
         testX, testY = create_dataset(test, look_back)
-
-        # concatenate on big training and test arrays
-        if flag:
-            trainX_long = trainX
-            trainY_long = trainY
-            testX_long = testX
-            testY_long = testY
-        else:
-            trainX_long = numpy.concatenate((trainX_long, trainX), axis = 0)
-            trainY_long = numpy.concatenate((trainY_long, trainY), axis = 0)
-            testX_long = numpy.concatenate((testX_long, testX), axis = 0)
-            testY_long = numpy.concatenate((testY_long, testY), axis = 0)
-        
-        flag = False
-
-    # reshape input to be [samples, time steps, features]
     
-    trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
-    testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
-    trainX_long = numpy.reshape(trainX_long, (trainX_long.shape[0], 1, trainX_long.shape[1]))
-    testX_long = numpy.reshape(testX_long, (testX_long.shape[0], 1, testX_long.shape[1]))
-    print(len(trainX_long))
+        # reshape input to be [samples, time steps, features]
+        trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
+        testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
 
-
-    # fit the LSTM network
-    model.fit(trainX_long, trainY_long, nb_epoch=50, batch_size=1, verbose=2)
+        # fit the LSTM network
+        model.fit(trainX, trainY, nb_epoch=5, batch_size=1, verbose=2)
+        
     
 ###################################################################################################