TrustMiner2/prediction.py

import numpy as np
import matplotlib.pyplot as plt
import pandas
import math
import sys
from keras import backend as K
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import GRU
from keras import optimizers as opt
from keras.layers import Dense
from keras.layers import Activation, Dropout
from keras.models import load_model
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
import copy
import random
import arimaPred as arim
import paper_plots as carlosplt


np.random.seed(7)
EPSILON = 10 ** -12

def moving_average(a, n) :
    ## code from Jaime
    ret = np.cumsum(a, dtype=float)
    ret[n:] = ret[n:] - ret[:-n]
    ret = np.concatenate((np.zeros(n-1), ret[n-1:]))
    return ret / n


# convert an array of values into a dataset matrix
# ATTENTION: THIS FUNCTION CHANGES SIZE OF INPUT
def create_dataset(original_dataset, dataset, num_steps, look_back=1, feat_num=1, extra_features=[]):
    dataX, dataY = [], []
    print('Making training length :', len(original_dataset), ' ; ', len(dataset))
    for i in range(len(dataset)-look_back-num_steps +1):
                a = []
                for j in range(i, i+look_back):
                    if feat_num>1:
                        a.append([original_dataset[j]]+extra_features.tolist())
                    else:
                        a.append([original_dataset[j]])
                dataX.append(a)
                mean = 0
                for j in range(num_steps):
                    mean += original_dataset[i+look_back+j]


                dataY.append(mean/num_steps)
    return np.array(dataX), np.array(dataY)

def test_model(pkg_name, original_dataset, dataset, scaler, num_steps, smoothing, batch_num, look_back, test_size, mixed, feat_num=1, extra_features=[]):
    ## mixed is a boolean. When True, the model trained on all the packages is used. When false, each package has its own model.
    
    totalX, totalY = create_dataset(original_dataset, dataset, num_steps, look_back, feat_num, extra_features)
    K.clear_session()

    if mixed:
        new_model = load_model('./models/all_packages'+str(num_steps) + '.h5')
    else:
        new_model = load_model('./models/' + pkg_name + '-' + str(num_steps) + 'smoothing' + '.h5')
    
    new_model.reset_states()
    print(len(totalX))
    totalPredict = new_model.predict(totalX[len(totalX)-batch_num:], batch_size = batch_num)
    del new_model
    
    #scaler = scalers[pkg_name]
    totalPredict = scaler.inverse_transform(totalPredict)
    totalPredict = totalPredict.flatten()
    total_LSTM = [0] * (len(dataset)-len(totalPredict)) + totalPredict.tolist()
    #print(len(total_LSTM))
    #print(total_LSTM)

    #totalY = totalY[18:]
    totalY = totalY.reshape(-1,1)
    totalY = scaler.inverse_transform(totalY)
    totalY = totalY.flatten()

    temp = dataset.reshape(-1,1)
    temp = scaler.inverse_transform(temp)
    temp = temp.flatten()
    
    if (math.fabs(totalY[-1]-temp[-1])>EPSILON):
        print("Possible fault!!", totalY[-1], temp[-1])


    return total_LSTM[:]

def predict_stationary(original, dataset, num_steps):
    prediction = dict()
    for pkg in dataset:
        data = dataset[pkg]
        pred = []
        i = 0
        for month in data:
            if i<num_steps:
                pred.append(0)
            else:
                pred.append(data[i-num_steps])
            i+=1
        prediction[pkg] = pred[:]
    return prediction

def predict_average(original, dataset, num_steps):
    prediction = dict()
    for pkg in dataset:
        o_data = dataset[pkg]
        average_list = []
        i = 0
        j = i
        flag = True
        for month in o_data:
            if(month==0 and flag):
                average_list.append(0)
                i+=1
                j=i
            else:
                flag = False
                if(len(o_data[i:j])<=num_steps):
                    average_list.append(0)
                else:
                    # average up until before the time
                    average = sum(o_data[i:j-num_steps])/len(o_data[i:j-num_steps])
                    average_list.append(average)
                j += 1
        prediction[pkg]=average_list[:]
    return prediction

def predict_Waverage(original, dataset, Lamd, num_steps):
    prediction = dict()
    for pkg in dataset:
        o_data = dataset[pkg]
        waverage_list = []
        i = 0
        j = i
        flag = True
        for month in o_data:
            if(month == 0 and flag):
                waverage_list.append(0)
                i += 1
                j = i
            else:
                flag = False
                if(len(o_data[i:j])<=num_steps):
                    waverage_list.append(0)
                else:
                    w_average = 0
                    weights = 0
                    jj = 0
                    local = o_data[i:j-num_steps]
                    for k in local:
                        w_average += k * math.exp(-(len(local) - jj - 1)/Lamd)
                        weights +=  math.exp(-(len(local) - jj - 1)/Lamd)
                        jj += 1
                    if (weights==0):
                        w_average = 0
                    else:
                        w_average = w_average/weights
                    waverage_list.append(w_average)
                j += 1
            prediction[pkg] = waverage_list[:]
    return prediction

def train_LSTM(original_data, data, num_steps, data_size, train_size, test_size, batch_num, model, Wsave, feat_num, look_back, pkg_name, scaler):

    model.set_weights(Wsave)
    ## Set the initial weights - remove if we want one model for all the packages
    train_original, test_original = original_data[0:train_size], original_data[train_size:len(original_data)]
    train, test = data[0:train_size], data[train_size:len(data)]
    if (len(original_data) != len(data)):
        return(1)
    print('Training and test size in months: ', len(train), ' ; ', len(test))

    trainX, trainY = create_dataset(train_original, train, num_steps, look_back)
    testX, testY = create_dataset(test_original, test, num_steps, look_back)
    
    # reshape input to be [samples, time steps, features]
    trainX = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], feat_num))
    testX = np.reshape(testX, (testX.shape[0], testX.shape[1], feat_num))

    print(len(trainX), len(testX))
    print(len(trainY), len(testY))
    
    trainY.reshape(-1,1)
    testY.reshape(-1,1)
    
    fig = plt.gcf()
    fig.clf()
    fig.show()
    fig.canvas.draw()

    training_steps = 400*25
    trainerror = []
    valerror=[]
    
    for j in range (training_steps):
        model.fit(trainX, trainY, epochs=1, batch_size=len(trainX), verbose=2, shuffle=False)
        model.reset_states()
        if(j%25==0):
            calc_errors(model, batch_num, original_data, data, num_steps, look_back, test_size, pkg_name, trainerror, valerror, scaler, feat_num, [])
            plt.plot(trainerror, color='blue')
            plt.plot(valerror, color='red')
            plt.title(pkg_name)
            fig.canvas.draw()
    try:
        model.save('./models/' + pkg_name + '-' + str(num_steps) + 'smoothing' + '.h5')
        #model.save('./models/all_packages'+str(num_steps) + '.h5')
        del model
    except OSError:
        model.save('./models/unknown-' + str(num_steps) + '.h5')
        del model

def train_LSTM_all(original_data, data, num_steps, data_size, train_size, test_size, batch_num, model, Wsave, feat_num, look_back, pkg_name, scaler, extra_features):

    train_original, test_original = original_data[0:train_size], original_data[train_size:len(original_data)]
    train, test = data[0:train_size], data[train_size:len(data)]
    if (len(original_data) != len(data)):
        return(1)
    print('Training and test size in months: ', len(train), ' ; ', len(test))

    trainX, trainY = create_dataset(train_original, train, num_steps, look_back, feat_num, extra_features)
    testX, testY = create_dataset(test_original, test, num_steps, look_back, feat_num, extra_features)
    
    # reshape input to be [samples, time steps, features]
    trainX = np.reshape(trainX, (trainX.shape[0], trainX.shape[1], feat_num))
    testX = np.reshape(testX, (testX.shape[0], testX.shape[1], feat_num))

    trainY.reshape(-1,1)
    testY.reshape(-1,1)
    
    trainerror = []
    valerror = []
    
    model.fit(trainX, trainY, epochs=1, batch_size=len(trainX), verbose=2, shuffle=False)
    model.reset_states()

    return 0

def calc_errors(model, batch_num, original_dataset, dataset, num_steps, look_back, test_size, pkg_name, trainerror, valerror, scaler, feat_num, extra_feature):

    totalX, totalY = create_dataset(original_dataset, dataset, num_steps, look_back, feat_num, extra_feature)

    totalX = totalX[len(totalX)-batch_num:]
    totalY = totalY[len(totalY)-batch_num:]

    model.reset_states()
    totalPredict = model.predict(totalX, batch_size = batch_num)

    totalPredict = scaler.inverse_transform(totalPredict)
    totalPredict = totalPredict.flatten()
    
    totalY = totalY.reshape(-1,1)
    totalY = scaler.inverse_transform(totalY)
    totalY = totalY.flatten()
    
    trainerror.append(mean_squared_error(totalPredict[50:-test_size], totalY[50:-test_size]))
    valerror.append(mean_squared_error(totalPredict[-test_size:], totalY[-test_size:]))

    return 0


def calc_errors_all(model, batch_num, original_data_in, data_in, num_steps, look_back, test_size, trainerror, valerror, scalers, feat_num, extra_features):
    temp1 = 0
    temp2 = 0
    train_temp = []
    test_temp = []

    for pkg in data_in:
        scaler = scalers[pkg]
        calc_errors(model, batch_num, original_data_in[pkg], data_in[pkg], num_steps, look_back, test_size, pkg, train_temp, test_temp, scaler, feat_num, extra_features[pkg]) ** 2

    for errors in train_temp:
        temp1 += errors ** 2
    
    for errors in test_temp:
        temp2 += errors ** 2

    trainerror.append(math.sqrt(temp1)/len(data_in))
    valerror.append(math.sqrt(temp2)/len(data_in))

    return 0

def predict_LSTM(original, dataset, num_steps, test_size, smoothing, first, do_train):
    # Model parameters
    # Do testing?
    do_test = True
    # Number of different models to train for
    models_num = 5
    # Look back steps
    look_back = 9
    # Number of lstm neurons
    num_neurons = look_back
    num_neurons2 = look_back
    # in case we want to add more features in the future
    feat_num = 1
    
    data_size = len(dataset[first])
    print(data_size)
    train_size = int(data_size - test_size)
    batch_num = train_size - num_steps - look_back + 1
    print("batch_num: ", batch_num)

    ## Create the NN with Keras
    model = Sequential()
    #model.add(LSTM(num_neurons, batch_input_shape = (batch_num, look_back, feat_num) , activation ='relu', dropout=0.4, stateful=True, return_sequences=True))
    model.add(LSTM(num_neurons, batch_input_shape = (batch_num, look_back, feat_num) , activation ='relu', dropout=0.5, stateful=True))
    #model.add(LSTM(num_neurons2, activation ='relu', dropout=0.4, stateful=True))
    model.add(Dense(1))
    Adam = opt.Adam(lr=0.001)
    model.compile(loss='mean_squared_error', optimizer=Adam)
    Wsave = model.get_weights()


    pred_LSTM = dict()
    data_in = dict()
    original_data_in = dict()
    scalers = dict()
    
    ## Let's start preparing our data
    for pkg in dataset:
        data = dataset[pkg]
        data = data.reshape(-1,1)
        scalers[pkg] = MinMaxScaler(feature_range=(0, 1))
        scalers[pkg].fit(data)
        ## We use scaler for each package seperately
        data = scalers[pkg].transform(data)
        data = data.flatten()
        original_data = original[pkg]
        original_data = original_data.reshape(-1,1)
        original_data = scalers[pkg].transform(original_data)
        original_data = original_data.flatten()
        data_in[pkg] = data
        original_data_in[pkg] = original_data

        ## Compute the total number of reported vulnerabilities in case we need it later
        total_sum = sum(original[pkg])

    ## Let's start with the training - if we want to train ofc...
    if do_train:
        ## Just a test to have one LSTM for all packages
        for i in range(1):
            for pkg in dataset:
                ## random selection for mixed training
                ## CHANGE for dedicated models
                #pkg = random.choice(list(dataset.keys()))
                data = data_in[pkg]
                original_data = original_data_in[pkg]
                train_LSTM(original_data, data, num_steps, data_size, train_size, test_size, batch_num, model, Wsave, feat_num, look_back, pkg, scalers[pkg])
        
    if do_test:
        for pkg in dataset:
            data = data_in[pkg]
            original_data = original_data_in[pkg]
            pred_LSTM[pkg] = test_model(pkg, original_data, data, scalers[pkg], num_steps, smoothing, batch_num, look_back, test_size, False)
    
    return pred_LSTM


def predict_LSTM_all(original, dataset, num_steps, test_size, smoothing, first, do_train):
    # Model parameters
    do_test = True
    # Look back steps
    look_back = 9
    feat_num = 1+len(dataset)
    # Number of lstm neurons
    num_neurons = feat_num + 10
    num_neurons2 = look_back
    # in case we want to add more features in the future
    extra_features = dict()
    ## Training steps
    training_steps = 600*25

    ## Use one extra feature to signal package identity
    i = 0
    for pkg in dataset:
        extra_features[pkg] = np.asarray([0]*i + [1] + [0]*(len(dataset)-i-1))
        extra_features[pkg].reshape(-1,1)




    
    data_size = len(dataset[first])
    print(data_size)
    train_size = int(data_size - test_size)
    batch_num = train_size - num_steps - look_back + 1
    print("batch_num: ", batch_num)

    ## Create the NN with Keras
    model2 = Sequential()
    #model2.add(Dense(units=num_neurons, activation='relu', batch_input_shape=(batch_num, look_back,feat_num)))
    model2.add(LSTM(num_neurons, batch_input_shape = (batch_num, look_back, feat_num) , activation ='relu', dropout=0.4, stateful=True, return_sequences=True))
    #model2.add(LSTM(num_neurons, batch_input_shape = (batch_num, look_back, feat_num) , activation ='relu', dropout=0.1, stateful=True))
    model2.add(LSTM(num_neurons, activation ='relu', dropout=0.1, stateful=True))
    #model2.add(Dense(num_neurons))
    model2.add(Dense(1))
    Adam = opt.Adam(lr=0.001)
    model2.compile(loss='mean_squared_error', optimizer=Adam)
    Wsave = model2.get_weights()


    pred_LSTM_all = dict()
    data_in = dict()
    original_data_in = dict()
    scalers = dict()
    
    ## Let's start preparing our data
    for pkg in dataset:
        data = dataset[pkg]
        data = data.reshape(-1,1)
        scalers[pkg] = MinMaxScaler(feature_range=(0, 1))
        scalers[pkg].fit(data)
        ## We use scaler for each package seperately
        data = scalers[pkg].transform(data)
        data = data.flatten()
        original_data = original[pkg]
        original_data = original_data.reshape(-1,1)
        original_data = scalers[pkg].transform(original_data)
        original_data = original_data.flatten()
        data_in[pkg] = data
        original_data_in[pkg] = original_data

        ## Compute the total number of reported vulnerabilities in case we need it later
        total_sum = sum(original[pkg])

    ## Let's start with the training - if we want to train ofc...
    if do_train:
        ## Just a test to have one LSTM for all packages
        fig = plt.gcf()
        fig.clf()
        fig.show()
        fig.canvas.draw()
        trainerror = []
        valerror=[]
        for i in range(training_steps):
            ## random selection for mixed training
            pkg = random.choice(list(dataset.keys()))
            data = data_in[pkg]
            original_data = original_data_in[pkg]
            train_LSTM_all(original_data, data, num_steps, data_size, train_size, test_size, batch_num, model2, Wsave, feat_num, look_back, pkg, scalers[pkg], extra_features[pkg])
            if(i%25==0):
                calc_errors_all(model2, batch_num, original_data_in, data_in, num_steps, look_back, test_size, trainerror, valerror, scalers, feat_num, extra_features)
                plt.plot(trainerror, color='blue')
                plt.plot(valerror, color='red')
                plt.title('Mixed')
                fig.canvas.draw()
        try:
            model2.save('./models/all_packages'+str(num_steps) + '.h5')
            del model2
        except OSError:
            model2.save('./models/unknown-' + str(num_steps) + '.h5')
        
    if do_test:
        for pkg in dataset:
            data = data_in[pkg]
            original_data = original_data_in[pkg]
            pred_LSTM_all[pkg] = test_model(pkg, original_data, data, scalers[pkg], num_steps, smoothing, batch_num, look_back, test_size, True, feat_num, extra_features[pkg])

    return pred_LSTM_all
    

def load_dataset_smoothed(src2month, pkglist, smoothing):
    temp = dict()
    original = dict()
    for pkg in pkglist:
        temp1 = np.asarray(src2month[pkg])
        temp2 = temp1
        # Smooth the time-series with a moving average
        temp1 = moving_average(temp1, n=smoothing)
        temp1 = temp1[smoothing:]
        temp2 = temp2[smoothing:]
        ## Cut off leading part
        temp[pkg] = temp1
        original[pkg] = temp2
    return (original, temp)


def print_all(data, pkglist):
    plt.figure(1)
    i = 1
    for pkg in pkglist:
        plt.subplot(2,5,i)
        plt.plot(data[pkg], label = pkg)
        plt.legend()
        i += 1
    plt.show()

def print_pred(data, pred):
    plt.plot(data, color='blue', label='reality')
    plt.plot(pred, color='red', label='model')
    plt.legend()
    plt.show()

def print_all_pred(data, pred, pkglist, test_size):
    carlosplt.pre_paper_plot(True)
    plt.figure(1)
    i = 1

    ## Build x axis
    quartersx = []

    for y in range(2,19):
        start = 1
        end = 5
        if y == 2:
            start = 2
        elif y == 19:
            end = 2
        for j in range(start,end):
            if j==1 and y%2==1:
                quartersx.append('\''+str(y).zfill(2))
            else:
                quartersx.append(' ')

    for pkg in pkglist:
        data_local = data[pkg]
        data_local = data_local.flatten()
        ax = plt.subplot(1,5,i)
        data_train = data_local[:-test_size]
        data_test = data_local[-test_size:]
        if pkg == 'firefox-esr':
            pkg_label = 'firefox'
        elif pkg == 'chromium-browser':
            pkg_label = 'chromium'
        elif pkg == 'openjdk-8':
            pkg_label = 'openjdk'
        else:
            pkg_label = pkg
        x_axis_train = []
        x_axis_test = []
        for j in range(len(data_train)):
            x_axis_train.append(j)
        for j in range(len(data_test)):
            x_axis_test.append(j+len(data_train))
        x_axis_all = x_axis_train + x_axis_test
        
        if i==5:
            train=ax.plot(x_axis_train, data_train, color = 'grey', label='real-tr')
            test=ax.plot(x_axis_test, data_test, color = 'coral', label = 'real-te')
            model=ax.plot(x_axis_all,pred[pkg], color ='blue', label='model')
            ax.legend()
        else:
            ax.plot(x_axis_train, data_train, color = 'grey')
            ax.plot(x_axis_test, data_test, color = 'coral')
            ax.plot(x_axis_all,pred[pkg], color ='blue')
        ax.spines['right'].set_visible(False)
        ax.spines['top'].set_visible(False)
        ax.set_title(pkg_label)
        plt.xticks(np.arange(1,len(pred[pkg]),3.0)+1,quartersx, rotation="vertical")
        i += 1
    carlosplt.post_paper_plot(True,True,True)
    plt.show()

def normalize_errors(errors_list, pkglist, dataset, test_size):
    for pkg in errors_list:
        maximum = np.amax(dataset[pkg][-test_size:])
        minimum = np.amin(dataset[pkg][-test_size:])
        norm = maximum-minimum
        if norm<0.1:
            norm = 0.1
        for d in [errors_list]:
            d[pkg] = d[pkg]/norm

def compute_errors_perpackage(prediction, dataset, test_size):
    temp_errors = dict()
    for pkg in prediction:
        temp_errors[pkg] = math.sqrt(mean_squared_error(prediction[pkg][-test_size:], dataset[pkg][-test_size:]))
    return temp_errors

def compute_errors_train_perpackage(prediction, dataset, test_size):
    temp_errors = dict()
    for pkg in prediction:
        temp_errors[pkg] = math.sqrt(mean_squared_error(prediction[pkg][:-test_size], dataset[pkg][:-test_size]))
    return temp_errors

def find_best_Lamd(original, dataset, num_steps, test_size):
    pred_Waverage_temp = dict()
    pred_Waverage = dict()
    errors = dict()
    best_errors = dict()
    best_lamdas = dict()
    dataset_temp = dict()

    pred_Waverage = predict_Waverage(original, dataset, 1, num_steps)
    errors[1] = compute_errors_train_perpackage(pred_Waverage, dataset, test_size)
    best_errors = errors[1]
    
    for pkg in dataset:
        best_lamdas[pkg] = 1
    
    for Lamd in range(1,100):
        pred_Waverage_temp = predict_Waverage(original, dataset, Lamd, num_steps)
        ## To compute best lamda
        errors[Lamd] = compute_errors_train_perpackage(pred_Waverage_temp, dataset, test_size)
        for pkg in pred_Waverage_temp:
            print(errors[Lamd][pkg])
            if errors[Lamd][pkg] < best_errors[pkg]:
                best_errors[pkg] = errors[Lamd][pkg]
                best_lamdas[pkg] = Lamd

    for pkg in dataset:
        pred_Waverage[pkg] = predict_Waverage(original, dataset, best_lamdas[pkg], num_steps)[pkg]

    print(best_lamdas)


    return pred_Waverage

def do_training_errors(predictions_list, errors, dataset, test_size):
    ## First for each package

    for method in predictions_list:
        temp_dict = dict()
        try:
            temp_dict = compute_errors_train_perpackage(predictions_list[method], dataset, test_size)
            errors[method] = temp_dict
        except:
            print('Predictions missing')


def do_testing_errors(predictions_list, errors, dataset, test_size):
    
    for method in predictions_list:
        temp_dict = dict()
        try:
            temp_dict = compute_errors_perpackage(predictions_list[method], dataset, test_size)
            errors[method] = temp_dict
        except:
            print('Predictions missing')

def calculate_rmse(errors):
    temp = 0
    for pkg in errors:
        temp += errors[pkg]*errors[pkg]

    temp = math.sqrt(temp)/len(errors)

    return temp

def calculate_mean(errors):
    temp = 0
    for pkg in errors:
        temp += errors[pkg]

    temp = temp/len(errors)

    return temp

def print_summary(training_errors, testing_errors):
    print('#'*80)
    print('***** REPORT *****')
    for method in training_errors:
        print(method)
        print('Training Errors rmse: ', '%.3f' % training_errors[method]['rmse'])
        print('Testing Errors rmse: ', '%.3f' % testing_errors[method]['rmse'])
        print('Training Errors mean: ', '%.3f' %training_errors[method]['mean'])
        print('Testing Errors mean: ', '%.3f' %testing_errors[method]['mean'])
        print('#'*80)
    
    return 0

def predict(src2month, k):

    pkglist=[]

    for pkg in src2month:
        if (sum(src2month[pkg])>50):
            pkglist.append(pkg)

    
    #pkglist = ['linux', 'firefox-esr', 'chromium-browser', 'icedove', 'wireshark', 'openjdk-8', 'mysql-transitional', 'php7.0', 'imagemagick', 'tcpdump']
    #pkglist = ['linux', 'firefox-esr', 'chromium-browser', 'icedove', 'openjdk-8']
    pkglist = ['icedove']
    first = pkglist[0]
    #pkglist = [first]
    # Number of months in the future
    num_steps = 9
    smoothing = num_steps
    # Test dataset size in months
    test_size = 18
    real_test_size = 18
    do_train = False

    dataset = dict()
    # Cut out end of 2018
    for pkg in pkglist:
        ## This is for training
        if do_train:
            src2month[pkg] = src2month[pkg][:-9-real_test_size]
        ## This is for experiments
        else:
            src2month[pkg] = src2month[pkg][test_size:-9]


    (original,dataset) = load_dataset_smoothed(src2month, pkglist, smoothing)
    # Each point of dataset is the mean of the same point and the previous smoothing-1 of the original
    num_packages = len(pkglist)

    # Print all smoothed time-series
    #print_all(dataset, pkglist)
    
    ## Make simple predictions (stationary, average, waverage)
    predictions_list = dict()
    predictions_list['stationary'] = predict_stationary(original, dataset, num_steps)
    predictions_list['average'] = predict_average(original, dataset, num_steps)
    predictions_list['Waverage'] = find_best_Lamd(original, dataset, num_steps, test_size)
    #predictions_list['LSTM'] = predict_LSTM(original, dataset, num_steps, test_size, smoothing, first, do_train)
    #predictions_list['LSTM_all'] = predict_LSTM_all(original, dataset, num_steps, test_size, smoothing, first, do_train)
    
    #print_all_pred(dataset, predictions_list['LSTM'], pkglist, test_size)
    #pkglist_new=['linux','firefox-esr', 'chromium-browser', 'icedove']
    pkglist_new = pkglist
    print_all_pred(dataset, predictions_list['Waverage'], pkglist_new, test_size)

    training_errors = dict()
    ## Dictionary of training errors e.g. training_errors['LSTM']['linux'] = XXX
    testing_errors = dict()
    ## Same for testing errors

    new_predictions_list = dict()

    ## For which packages to compute the error?
    for method in predictions_list:
        new_predictions_list[method] = dict()
        for pkg in predictions_list[method]:
            if (sum(src2month[pkg])>200):
                new_predictions_list[method][pkg] = predictions_list[method][pkg]


    print(new_predictions_list)
    
    do_training_errors(new_predictions_list, training_errors, dataset, test_size)
    do_testing_errors(new_predictions_list, testing_errors, dataset, test_size)

    ## Now among the packages again rmse. But first we normalize. Choose whether we want this or not

    for method in training_errors:
        normalize_errors(training_errors[method], pkglist, dataset, test_size)
        normalize_errors(testing_errors[method], pkglist, dataset, test_size)

    for pkg in training_errors['average']:
        print('#'*80)
        print(pkg)
        print('Training errors:')
        temp_list = []
        for method in training_errors:
            string = method + ': ' + str(training_errors[method][pkg]) + ' , '
            temp_list.append(string)

        print(temp_list)
        
        temp_list = []
        for method in training_errors:
            string = method + ': ' + str(testing_errors[method][pkg])
            temp_list.append(string)

        print('Testing errors:')
        print(temp_list)

    ## Now it is time for the rmse among the packages
    for method in testing_errors:
        testing_errors[method]['rmse'] = calculate_rmse(testing_errors[method])
        testing_errors[method]['mean'] = calculate_mean(testing_errors[method])
        training_errors[method]['rmse'] = calculate_rmse(training_errors[method])
        training_errors[method]['mean'] = calculate_mean(training_errors[method])

    
    print_summary(training_errors, testing_errors)
    
    return