TrustMiner2/old/apt-sec.mbox

#!/usr/bin/perl

use strict;
use warnings;

use Storable; # persistant storage
use LWP::Simple; # libwww-perl
use Config::Auto; # libconfig-auto-perl
use Time::ParseDate; # libtime-modules-perl
use Linux::Distribution qw(distribution_name distribution_version); # liblinux-distribution-perl
#use IO::Compress::Bzip2 qw(bzip2 $Bzip2Error);
use IO::Uncompress::Bunzip2 qw(bunzip2 $Bunzip2Error);
#use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
use Digest::MD5 qw(md5_hex);
use POSIX qw(mktime);
#use DB_File; # berkely DB backend for SHA1 list (2.3GB hash table...)


use feature "switch";

# don't try to parse perl in configs
$Config::Auto::DisablePerl = 1;

#print "This is libwww-perl-$LWP::VERSION\n";

# load user config
my $config = Config::Auto::parse("apt-sec.conf");

# global variables
my $secperday = 60*60*24;
my $now = mktime(localtime()); # look, we're atomic!  :-)
my $verbosity = 1;
my @months = ("January", "February", "March", "April", "May", "June", "July",
	"August", "September", "October", "November", "December");

# global lookup tables
my %h_dsatable; # map dsa_id   => dsa
my %h_cvetable; # map cve_id   => (rel-date, time-to-fix, score1, score2, score3))
my %h_src2dsa;  # map src-name => dsa_id
my %h_dsa2cve;  # map dsa_id   => cve_id
my %h_src2mtbf; # map src-name => MTBFstats

my %h_deb2pkg;  # map deb-name => pkg-name
my %h_pkg2src;  # map pkg-name => src-name
my %h_sha1map;  # map sha1sum  => file-path

my %h_state;    # remember what we've already parsed/downloaded

my $dsatable = \%h_dsatable;
my $cvetable = \%h_cvetable;
my $src2dsa  = \%h_src2dsa;
my $dsa2cve  = \%h_dsa2cve;
my $src2mtbf = \%h_src2mtbf;

my $deb2pkg  = \%h_deb2pkg;
my $pkg2src  = \%h_pkg2src;
my $sha1map  = \%h_sha1map;

my $state    = \%h_state;

sub detect_distribution {
	my $dist = "";
	my $version = "";
	
	$dist = distribution_name();
	$version = distribution_version();
	
	if ( $dist eq "") {
		print "Error: Distribution unknown, not LSB conform, bailing out.\n";
		exit;
	}
	
	if ($state->{"vendor"} eq "" || $state->{"vendor"} ne $dist) {
		print "Detected $dist distribution, version $version\n";
		$state->{"vendor"} = $dist;
	}
}

## logging
sub msg {
	my $lvl = shift;
	my $msg = shift;
	print "$msg\n" unless ($lvl > $config->{"loglevel"});
}

## load state, different from DBs in that we always need it
sub load_state {
	my $cache = $config->{cache_dir};
	my $err = 0;
	
	eval { $state = retrieve($cache . "state"); } or do {
		## init with default state
		$state->{"next_dsa"} = 11; ## can't parse DSAs earlier than that
		$state->{"Packages"} = "";
		$state->{"Sources"} = "";
		$state->{"Sha1Sums"} = "";
		$state->{"vendor"} = "";
		$err++;
	};
	return 1-$err;
}

## save state, different from DBs in that we always need it
sub save_state {
	my $cache = $config->{cache_dir};
	eval {
		store($state,     $cache . "state");
		return 1;
	} or do {
		print "Failed to save state, check permissions:\n";
		print "$@";
	};
}
	

## persistant storage
sub load_DBs {
	my $cache = $config->{cache_dir};
	my $err = 0;
	
	eval {
	 	$dsatable  = retrieve($cache . "dsatable");
		$cvetable  = retrieve($cache . "cvetable");
		$src2dsa   = retrieve($cache . "src2dsa");
		$dsa2cve   = retrieve($cache . "dsa2cve");
		$src2mtbf  = retrieve($cache . "src2mtbf");
	} or do {
		#delete $dsatable->{$_} foreach (keys %$dsatable);
		#delete $cvetable->{$_} foreach (keys %$cvetable);
		#delete $src2dsa->{$_}  foreach (keys %$src2dsa);
		#delete $dsa2cve->{$_}  foreach (keys %$dsa2cve);
		#delete $src2mtbf->{$_} foreach (keys %$src2mtbf);
	 	%$dsatable  = ();
		%$cvetable  = ();
		%$src2dsa   = ();
		%$dsa2cve   = ();
		%$src2mtbf  = ();
		$state->{"next_dsa"} = 11; ## can't parse DSAs earlier than that
		$err++;
	};
	eval { $deb2pkg  = retrieve($cache . "deb2pkg");
	} or do {
		$state->{"Packages"} = "";
		%$deb2pkg=();
		$err++;
	};
	eval { $pkg2src  = retrieve($cache . "pkg2src");
	} or do {
		$state->{"Sources"} = "";
		%$pkg2src=();
		$err++;
	};
	
	eval { 
		$sha1map  = retrieve($cache . "sha1map");
	} or do {
		$state->{"Sha1Sums"} = "";
		%$sha1map=();
		$err++;
	};
	
	return 1-$err;
}


## persistant storage
sub save_DBs {
	my $cache = $config->{cache_dir};

	eval {
		# program status data
		store($state,     $cache . "state");
		# parsed/evaluated security data
		store($dsatable,  $cache . "dsatable");
		store($cvetable,  $cache . "cvetable");
		store($src2dsa,   $cache . "src2dsa");
		store($dsa2cve,   $cache . "dsa2cve");
		store($src2mtbf,  $cache . "src2mtbf");
		# parsed Debian packages data
		store($deb2pkg,   $cache . "deb2pkg");
		store($pkg2src,   $cache . "pkg2src");
		store($sha1map,   $cache . "sha1map");
		
		return 1;
	} or do {
		print "Failed to save cache file(s), check permissions:\n";
		print "$@";
	};
}

## Parse DSA data and return array
## (src-pkg-name date (CVE-id)*)
sub parseDSA {
	my $dsa = shift or die "No advisories to parse?!";
	
	my @dsa_names;
	my @dsa_CVEs;
	my $dsa_date;
	my $dsa_type;
	my @tmp;
	my $tmp_date;
	
	msg 10, "Trying to parse DSA..";

	my @lines = split(/\n/,$dsa);
	

	LINE: foreach my $line (@lines) {
		
		# Date Reported -> $dsa_date
		unless ($dsa_date) {
			#print "look for date in $line\n";
			if ($line =~ /^Date:\s+\w+,\s+(\d+)\s+(\w+)\s+(\d+)\s+\d+:\d+:\d+/) {
				$dsa_date = parsedate("$1 $2 $3");
			}
# parse date from body: contains many flaws, mail header seems always accurate
#			if ($line =~ /^(\w+)\s+(.+),\s+(\d+)\s*$/ ||
#			$line =~ /^(\w+)\s+(.+),\s+(\d+)\s+http:\/\/www.debian.org\/security/) {
#				print "using body date: $1 $2, $3\n";
#				$dsa_date = parsedate("$1 $2, $3");
#				if (abs($tmp_date - $dsa_date) > 86401 && $1 . $2 ne "December1st"	&&
#				 	$1 . $3 ne "January2004" && $1.$3 ne "Aug2005" && $1.$3 ne "July2005" && $1.$2 ne "August17th" &&
#				  $1.$2 ne "August25th") {
#					$dsa_date = $tmp_date if ($1 eq "XXXXX");
#					next LINE if ($1 eq "XXXXX");
#					print "\nDate mismatch:\n";
#					print localtime($tmp_date);
#					print "\n";
#					print localtime($dsa_date);
#					print "\n";
#					die;
#				}
#			}
			next LINE;
		}
	
		# Affected Packages -> @dsa_names
		unless (@dsa_names) {
			#print "look for name in $line\n";
			if ($line =~ m/^Vulnerability\s*:/) {
				@dsa_names = @tmp;
				@tmp=();
			}
			if ( @tmp && $line =~ /^\s+(.*)/) {
				foreach my $w (split(/(,|\s+)/, $1)) {
					$w =~ s/,\s*//;
					push @tmp, $w;
				}
			}
			if ($line =~ /^Package\s*:\s+(.*)/) {
				foreach my $w (split(/(,|\s+)/, $1)) {
					$w =~ s/,\s*//;
					push @tmp, $w;
				}
			}
			next LINE;
		}

		# Security database references (CVEs) -> @dsa_CVEs
		unless (@dsa_CVEs) {
			#print "look for cve in $line\n";
			if ($line =~ m/^CVE\ IDs?\s+:\s(.*)/i ||
			    $line =~ m/^CVE\ Id\(s\)\s+:\s(.*)/i) {
				foreach my $w (split(/\s+/, $1)) {
					$w =~ s/,\s*//;
					push @tmp, $w;
				}
			}
			if ( @tmp && $line =~ /^\s+(.*)/) {
				foreach my $w (split(/\s+/, $1)) {
					$w =~ s/,\s*//;
					push @tmp, $w;
				}
			}
			# anything else is considered end of entry
			@dsa_CVEs = @tmp;
			@tmp=();
			last if ($line =~ m/^$/);
			next LINE;
		}
	}
	
	print "\n\nNames: @dsa_names\n";
	print "Date:  $dsa_date\n";
	print "CVEs:  @dsa_CVEs\n";
	return (\@dsa_names, $dsa_date, \@dsa_CVEs);
}

## Parse USN data and return array
## (src-pkg-name date (CVE-id)*)
sub parseUSN {
	my $usn = shift or die "No advisories to parse?!";
	
	my @usn_names;
	my @usn_CVEs;
	my $usn_date;
	my $usn_type;
	my @tmp;
	
	msg 10, "Trying to parse USN..";

	my @lines = split(/\n/,$usn);
	LINE: foreach my $line (@lines) {
		
		# Date Reported -> $dsa_date
		unless ($usn_date) {
			#print "Looking for date\n";
			#print "Line: $line\n";
			if ($line =~ /^Ubuntu\ Security\ Notice\ .*\d+-\d+\s+(\w+)\s+(\d+),\s+(\d+)/) {
				$usn_date = parsedate("$1 $2, $3");
			}
			next LINE;
		}
	
		# Affected Packages -> @dsa_names
		unless (@usn_names) {
			#print "Looking for name\n";
			#print "Line: $line\n";
			if ($line =~ m/^CVE-/ || $line =~ m/^CAN-/ || $line =~ m/^http.*launchpad.net\/bugs/|| $line =~ m/^===========/) {
				@usn_names = @tmp;
				@tmp=();
				$usn_type = pop @usn_names;
			} else {
				foreach my $w (split(/\s+/, $line)) {
					$w =~ s/,\s*//;
					#print "Adding: $w\n";
					push @tmp, $w;
				}
				next LINE;
			}
		}

		# Security database references (CVEs) -> @dsa_CVEs
		unless (@usn_CVEs) {
			#print "Looking for cve\n";
			#print "Line: $line\n";
			if ($line =~ m/^===========/) {
				@usn_CVEs = @tmp;
			} else {
				foreach my $w (split(/\s+/, $line)) {
					$w =~ s/,\s*//;
					push @tmp, $w;
				}
			}
		}
	}
	
	#print "Names: @usn_names\n";
	#print "Date:  $usn_date\n";
	#print "CVEs: @usn_CVEs\n";
	return (\@usn_names, $usn_date, \@usn_CVEs);
}

## Get details of given CVE entry from NIST DB
sub fetchCVE {
	my $cve_id = shift;

	## print-only CVEs
	#print "CVE_ID: $cve_id\n";
	#return "";

	msg 10, "  Fetching $cve_id";
	
	$cve_id =~ s/^CAN/CVE/;
	
	my $url=	$config->{"cve_base_url"} . $cve_id;
	my $cve = get $url;
	#die "  Unable to fetch CVE $cve_id" unless defined $cve;
	unless (defined $cve) {
		print "Failed to download CVE: $url\n";
		return "";
	}
	
	# Check for error pages: referenced but unpublished CVEs :-/
	if ($cve =~ /is\ valid\ CVE\ format,\ but\ CVE\ was\ not\ found/) {
		msg 5, "  $cve_id does not exist in NIST DB\n";
		$cve = "";
	}
	
	return $cve;
}

## Static map to correct errors in DSAs
## Return fixed list of CVE IDs or 0 to skip DSA
sub fixDSAquirks {
	my $dsa_id = shift;
	my $dsa_state = shift;
	
	my @new_names = @{@$dsa_state[0]};
	my $new_date = @$dsa_state[1];
	my @new_cves = @{@$dsa_state[2]};
	
	## These DSAs are totally screwed up..
	given ($dsa_id) {
		when ($_ eq "DSA 310-1") {
		 	@new_cves = ("CVE-2003-0385");
		}
		when ($_ eq "DSA-045-2") {
		 	@new_cves = ("CVE-2001-0414");
		}
		when ($_ eq "DSA-045-1") {
		 	@new_cves = ("CVE-2001-0414");
		}
		when ($_ eq "DSA 745-1") {
		 	@new_cves = ("CVE-2010-2155", "CVE-2009-4882");
		}
		when ($_ eq "DSA-1931-1") {
		 	@new_cves = ("CVE-2009-0689", "CVE-2009-2463");
		}
		when ($_ eq "DSA-1941-1") {
		 	@new_cves = ("CVE-2009-0755", "CVE-2009-3903", "CVE-2009-3904",
			             "CVE-2009-3905", "CVE-2009-3606", "CVE-2009-3607",
									 "CVE-2009-3608", "CVE-2009-3909", "CVE-2009-3938");
		}
		when ($_ eq "DSA-2056-1") {
		 	@new_cves = ("CAN-2005-1921", "CAN-2005-2106");
		}
		when ($_ eq "DSA 1095-1") {
		 	@new_cves = ("CVE-2006-0747", "CVE-2006-1861", "CVE-2006-2661");
		}
		when ($_ eq "DSA-2092-1") {
		 	@new_cves = ("CVE-2010-1625", "CVE-2010-1448", "CVE-2009-4497");
		}
		when ($_ eq "DSA 1284-1") {
		 	@new_cves = ("CVE-2007-1320", "CVE-2007-1321", "CVE-2007-1322", "CVE-2007-2893", "CVE-2007-1366");
		}
		when ($_ eq "DSA 1070-1") {
			$_ =~ s/CVE-2005-0528/CVE-2003-0985/ foreach (@new_cves);
		}
	};
	
	return (\@new_names, $new_date, \@new_cves);
}

## static map to correct errors in USNs
## returns fixed list of CVE IDs or 0 to skip DSA
sub fixUSNquirks {
	my $usn_id = shift;
	my $usn_state = shift;
	
	my @new_names = @{@$usn_state[0]};
	my $new_date = @$usn_state[1];
	my @new_cves = @{@$usn_state[2]};
	
	## These DSAs are totally screwed up..
	given ($usn_id) {
		when ($_ eq "USN-6-1") {
			@new_names = ("postgresql");
		}
	};
	
	return (\@new_names, $new_date, \@new_cves);
}

## Get CVE severity rating and report date and return
## (date base-score impact-score exploit-score)
sub parseCVE {
	my $cve_id = shift;
	my $cve_date;
	my $cve_base;
	my $cve_impact;
	my $cve_exploit;

	my $cve = fetchCVE($cve_id);
	if ($cve eq "") {
		# No details means we assume highest score, but immediate fix.
		# There's not much justification for this, except CVSS also
		# assumes highest score..
		return (0, 10, 10, 10);
	}
 
	# Reporting Date -> $cve_date
	$cve =~ /<span class="label">Original\ release\ date:<\/span>(\d+)\/(\d+)\/(\d+)<\/div>/;
	$cve_date = parsedate("$3-$1-$2");
	die "  Unable to extract date from CVE" unless defined $cve_date;

	# Base Score -> $cve_base
	unless ($cve =~ /CVSS\sv2\sBase\sScore:<\/span><a\shref=".*>(.*)<\/a>\s\(\w+\)/) {
		die "  Unable to parse CVSSv2 Base Score for $cve_id";
	}
	$cve_base = $1;

	# Impact Subscore -> $cve_impact
	unless ($cve =~ /<div\ class="row"><span\ class="label">Impact\ Subscore:<\/span>\s+(.*)<\/div>/) {
		die "  Unable to parse CVSSv2 Impact Score for $cve_id";
	}
	$cve_impact = $1;

	# Exploitability Subscore -> $cve_exploit
	unless ($cve =~ /<div\ class="row"><span\ class="label">Exploitability\ Subscore:<\/span>\s+(.*)<\/div>/) {
		die "  Unable to parse CVSSv2 Exploitability Score for $cve_id";
	}
	$cve_exploit = $1;

	#print "$cve_base\n";
	#print "$cve_impact\n";
	#print "$cve_exploit\n";
	return ($cve_date, $cve_base, $cve_impact, $cve_exploit);
}

## Fetch current Packages, Sources and sha1sums files
## These are needed to find CVE stats by sha1sums/pkg-names
## Only Sha1Sums is custom generated, others are from Debian.
## FIXME: Server might do on-the-fly gzip (but should not for bzip2)
## Return: 1 on success, to signal that new parsing is needed.
sub fetchMeta {
	my $file = shift;
	my $urlbase =	$config->{"pkg_base_url"};
	my $dir = $config->{"cache_dir"};
	
	msg 10, "Checking meta file $file..";
	
	my $bzFile = $file . ".bz2";
	my $url = $urlbase . $bzFile;
	my $ret = mirror $url, $dir . $bzFile; # download if new

	if ($ret == 200) {
		
		# check if the file actally changed..
		open(FILE, $dir . $bzFile) or die "Can't open '$dir$bzFile': $!";
		binmode(FILE);
		my $stamp = Digest::MD5->new->addfile(*FILE)->hexdigest;
		
		if ($state->{$file} eq $stamp) {
			print "  unchanged..\n";
			return 0;
		} else {
			$state->{$file} = $stamp;
		}

		# file seems new, unpack for parsing..(TODO: should keep in $tmp..)
		bunzip2 $dir . $bzFile => $dir . $file#, AutoClose => 1
			or die "bunzip2 failed: $Bunzip2Error\n";
		return 1; # file changed
	} elsif ($ret != RC_NOT_MODIFIED ) {
		print "HTTP error code: $ret when fetching $url\n";
		return 0; # no updates
	}
}

## Parse dpkg Packages file, create map deb-name->pkg-name
sub parsePackages {
	my $pkgfile = shift;
	my %deb2pkg;
	my $pkgname;
	my $dir = $config->{"cache_dir"};
	
	msg 10, "Parsing Packages file...";
	
	$pkgfile = $dir . $pkgfile;
	open (PKG, "< $pkgfile");
	my @lines = <PKG>;
	
	LINE: foreach my $line (@lines) {
		if ($line =~ /^Package:\ (.+)/) {
			$pkgname = $1;
		} elsif ($line =~ /^Filename:\ .*\/(\S+)/) {
			$deb2pkg{$1} = $pkgname;
		} else {
			next LINE;
		}
	}
	#print map { "$_ => $deb2pkg{$_}\n" } keys %deb2pkg;
	return \%deb2pkg;
}

## Parse dpkg Sources file, create map pkg-name->src-name
sub parseSources {
	my $srcfile = shift;
	my $dir = $config->{"cache_dir"};
	my %pkg2src;
	my $srcname;
	my @pkgnames;
	my $checklinecontinuation=0;
	
	msg 10, "Parsing Sources file...";
	
	$srcfile = $dir . $srcfile;
	open (SRC, "< $srcfile");
	my @lines = <SRC>;
	
	LINE: foreach my $line (@lines) {
		## sometimes, list of binary pkgs has newline, so we need to check next line..
		if ($checklinecontinuation == 1) {
			unless ($line =~ /^[[:alpha:]]+:\ /) {
				@pkgnames = split(/,\ /, $line);
				foreach my $pkg (@pkgnames) {
					$pkg2src{$pkg} = $srcname;
				}
			} else {
				$checklinecontinuation = 0;
			}
			next LINE;
		}
		if ($line =~ /^Package:\ (.+)/) {
			$srcname = $1;
		} elsif ($line =~ /^Binary:\ (.+)/) {
			@pkgnames = split(/,\ /, $1);
			foreach my $pkg (@pkgnames) {
				$pkg2src{$pkg} = $srcname;
			}
			$checklinecontinuation = 1;
		} else {
			next LINE;
		}
	}
	#print map { "$_ => $pkg2src{$_}\n" } keys %pkg2src;
	return \%pkg2src;
}

## Parse Sha1Sums file. Format: "sha1sum::deb-name::unix-file-path"
## Create 2 maps: sha1sum->file, file->deb-name
sub parseSha1Sums {
	my $sha1file = shift;
	my %sha1map;
	my $dir = $config->{"cache_dir"};
	
	msg 10, "Parsing Sha1Sums file...";
	
	$sha1file = $dir . $sha1file;
	open (SHA, "< $sha1file");
	my @lines = <SHA>;
	LINE: foreach my $line (@lines) {
		if ($line =~ /^(\w+)::(\S+)$/) {
			# It seems Perl can't handle that, needs around 4GB RAM :-/
			#$sha1map->{$1} = $3;
			#$filemap->{$3} = $2;
			$sha1map{$1} = $2;
		} else {
			die "Sha1Sums parse error, line reads: \n>>$line<<";
		}
	}
	return \%sha1map;
}

## Parse local dpkg status, return list of debs
sub parseStatus {
	my $stsfile = shift;
	my @pkglist;
	my $pkgname;
	
	msg 10, "Parsing dpkg status..";
	
	open (PKG, "< $stsfile");
	my @lines = <PKG>;
	
	LINE: foreach my $line (@lines) {
		if ($line =~ /^Package:\ (.+)/) {
			$pkgname = $1;
		} elsif ($line =~ /^Status:.*installed/) {
			push @pkglist, $pkgname;
		} else {
			next LINE;
		}
	}
	return \@pkglist;
}

sub parseAdvisory {
	my $adv = shift;
	
	given ($state->{"vendor"}) {
		when ($_ eq "debian") { return parseDSA($adv); }
		when ($_ eq "ubuntu") { return parseUSN($adv); }
#		when ($_ eq "redhat") { return checkRHSA; }
		default               { die "Unsupported distribution $_"; }
	};
}

sub fixAdvisoryQuirks {
	my @arg = @_;
	given ($state->{"vendor"}) {
		
		when ($_ eq "debian") { return fixDSAquirks(@arg); }
		when ($_ eq "ubuntu") { return fixUSNquirks(@arg); }
#		when ($_ eq "redhat") { return checkRHSA; }
		default               { die "Unsupported distribution $_"; }
	};
}

## Update internal vuln. DB with new Advisory info
## Creates CVEtable for MTBF computation:
##    ( cve-id => (date, delay, score1, score2, score3))
sub updateCVETables {
	my $id   = shift; # Advisory to merge into tables
	my @cvestats;

	msg 10, "Updating vulnerability database with advisory ".$state->{"vendor"}."/$id\n";

	my $adv = $dsatable->{$id};
	my @dsastats = parseAdvisory($adv);
	
	#print "\nADV-Stats:\n";
	#print "@{$dsastats[0]}\n";
	#print "@{$dsastats[2]}\n";
	
	## fix DSAs that don't contain correct CVE refs
	@dsastats = fixAdvisoryQuirks($id, \@dsastats);
	#my $cve_ids = $dsastats[2];
	
	#print "\nADV-Stats:\n";
	#print "@{$dsastats[0]}\n";
	#print "@{$dsastats[2]}\n";
	
	
	foreach my $srcpkg (@{$dsastats[0]}) {
		push @{$src2dsa->{$srcpkg}}, $id;
		push @{$dsa2cve->{$id}}, @{$dsastats[2]};
	}

	foreach my $cve_id (@{$dsastats[2]}) {
		@cvestats = parseCVE($cve_id);

	 	if ($cvestats[0] > $dsastats[1] || $cvestats[0] == 0) {
			$cvestats[0] = $dsastats[1];
		}

		my @cvedata = ( $cvestats[0], $dsastats[1]-$cvestats[0],
			              $cvestats[1], $cvestats[2], $cvestats[3] );

		$cvetable->{$cve_id} = \@cvedata;
	}
}

## Check for updates on Package information
sub aptsec_update {
		
	if (fetchMeta("Packages")) {
		$deb2pkg = parsePackages("Packages");
	}

	if (fetchMeta("Sources")) {
		$pkg2src = parseSources("Sources");
	}

	if (fetchMeta("Sha1Sums")) {
		$sha1map = parseSha1Sums("Sha1Sums");
	}
	
	#readAdvisories($last)
}

## find list of src pkgs from bin pkgs based on pkg2src
sub resolvePkg2Src {
	my $pkglist = shift;
	my @srclist;
	my %tmp;
	my $srcpkg;

	foreach my $pkg (@$pkglist) {
		$srcpkg = $pkg2src->{$pkg};
		 
		if (defined $srcpkg) {
			$tmp{$pkg2src->{$pkg}} = 1;
		} else {
			msg 5, "Could not find source package for: $pkg";
		}
	}
	
	@srclist = keys %tmp;
	
	return \@srclist;
}


## compute and store MTBF, MTBR and Scores of each src pkg
## uses: @cvestats = (date base-score impact-score exploit-score)
## output: %src2mtbf = (srcpkg=> (begin, num, delaysum, scoresum, maximpact))
sub processCVEs {
	my @pkgs = shift;
	@pkgs = keys %$src2mtbf unless @pkgs;

	foreach my $pkg (@pkgs) {
		my @stats = ($now, 0, 0, 0, 0);
		foreach my $dsa_id (@{$src2dsa->{$pkg}}) {
			foreach my $cve_id (@{$dsa2cve->{$dsa_id}}) {
			 	if ($stats[0] > $cvetable->{$cve_id}[0]) {
					$stats[0] = $cvetable->{$cve_id}[0];
				}
				$stats[1]++;
				$stats[2]+= $cvetable->{$cve_id}[1];
				$stats[3]+= $cvetable->{$cve_id}[2];
			 	if ($stats[4] < $cvetable->{$cve_id}[3]) {
					$stats[4] = $cvetable->{$cve_id}[3];
				}
			}
		}
		# TODO: can only save for >=1?
		if ($stats[1] > 1) {
			$src2mtbf->{$pkg} = \@stats;
		}
	}
}

## Use local system status(dpkg DB) for printing system status report
sub aptsec_system {
	my $pkglist = parseStatus "/var/lib/dpkg/status";
	my $srclist = resolvePkg2Src $pkglist;
	
	printSystemStats($srclist);
}

## Print 'trustworthiness' of a set(system) of src packages
sub printSystemStats {
	my $srclist = shift;
	my $mtbf=0;
	my $mtrr=0;
	my $num=0;
	
	# we can assume that pkgs are independent (right?)
	PKG: foreach my $pkg (@$srclist) {
		my $rstats = $src2mtbf->{$pkg} or next PKG;
		my @stats = @$rstats;
		
		$mtbf+=1/(($now-$stats[0])/$stats[1]/$secperday);
		$mtrr+=$stats[2]/$stats[1]/$secperday;
		
		if ($verbosity > 0) {
			printf "MTBF: %4d, MTRR: %4d,\tPkg: %s\n",
			       ($now-$stats[0])/$stats[1]/$secperday,
			       $stats[2]/$stats[1]/$secperday,
			       $pkg;
				 }
		$num++;
	}
	
	printf "\n";
	printf "Packages with past vulnerabilities installed: %d\n", $num;
	printf "System MTBF: %5.1f days per failure\n", 1/$mtbf;
	printf "System MTRR: %5.1f days\n", $mtrr/$num;
	printf "\n";
}

## show info on a single src pkg, resolv to src if needed
sub aptsec_show {
	my $pkg = shift;
	
	unless (defined $pkg) {
		aptsec_help();
		exit;
		
	}
	
	if (!($src2dsa->{$pkg}) && $pkg2src->{$pkg}) {
		print "Resolving $pkg to $pkg2src->{$pkg}\n";
		$pkg = $pkg2src->{$pkg};
	}
	
	print "The following binary packages are created from $pkg:\n";
	foreach (keys %$pkg2src) {
		print "$_\n" if ($pkg2src->{$_} eq $pkg);
	}

	unless ($src2dsa->{$pkg} || @{$src2mtbf->{$pkg}}) {
		print "No vulnerabilities found for source package $pkg.";
		exit;
	}
	
	print "\nInformation on package $pkg:\n\n";
	
	foreach my $dsa_id (sort @{$src2dsa->{$pkg}}) {
		print "DSA: DSA-$dsa_id\n";
		foreach my $cve_id (@{$dsa2cve->{$dsa_id}}) {
			my ($sec,$min,$hrs,$day,$mon,$yr) = localtime($cvetable->{$cve_id}[0]);
			printf "%s: Base Score: %04.1f, %02d.%02d.%04d\n",
		 	       $cve_id, $cvetable->{$cve_id}[2], $day, $mon, $yr+1900;
		}
	}
	my @stats = @{$src2mtbf->{$pkg}};
	my ($sec,$min,$hrs,$day,$mon,$yr) = localtime($stats[0]);
	print  "\nOverall vulnerability stats: \n";
	printf "  First one reported:    %02d.%02d.%04d\n", $day, $mon, $yr+1900;
	printf "  Total vulnerabilities: %d\n", $stats[1];
	printf "  Average Base Score:    %04.2f\n", $stats[3]/$stats[1];
	printf "  Highest Impact Score:  %d\n", $stats[4];
	printf "  MTBF in days:          %.2f\n", ($now-$stats[0])/$stats[1]/$secperday;
	printf "  MTRR in days:          %.2f\n\n", $stats[2]/$stats[1]/$secperday;
}

sub aptsec_help {
	print "\n";
	print "Usage:\n";
	print "\n";
	print "help       This cruft\n";
	print "update     Update vulnerability databases\n";
	print "system     Compute expected failure rates for local system\n";
	print "show <pkg> Show failure rates and vulnerability statistics for <pkg>\n";
	print "\n";
}

## Print system status report from component(files) measurements (sha1sums)
## Expected input format is Linux IMA. We assume input was validated.
##
## Note: aptsec_system(), considers *reportedly installed* packages, while this
## one looks at *actually loaded* software that influenced the CPU since bootup.
sub aptsec_attest {
	my $sha1file = shift;
	my %tmp;
	my $pkg;
	my @pkglist=();
	

	$sha1file = "/sys/kernel/security/ima/ascii_runtime_measurements" unless ($sha1file);
	
	open (SHA, "< $sha1file") or die "Unable to open file $sha1file";
	my @lines = <SHA>;
	LINE: foreach my $line (@lines) {
		if ($line =~ /[0-9]{2}\ \w{40}\ ima\ (\w+)\ (\S+)/) {
			if ($sha1map->{$1}) {
				$tmp{$sha1map->{$1}} = 1;
			} else {
				print "Unknown measured file: $1 $2\n";
			}
		} else {
			print "Failed to parse attestation data input from $sha1file\n";
		}
	}
	
	foreach my $deb (keys %tmp) {
		if ($pkg = $deb2pkg->{$deb}) {
			push @pkglist, $pkg;
		}
	}
	my $srclist = resolvePkg2Src (\@pkglist);
	printSystemStats($srclist);
}

# read USN from stdin, assuming mbox format (separated by "^From ")
sub readDSA {
	my $adv="";
	my $id="";
	my %newAdv;
	
	msg 5, "Reading new DSA advisory...";
	
	my @lines = <STDIN>;
	LINE: foreach my $line (@lines) {
		unless ($line =~ m/^From\ /) {
			if ($line =~ /^Debian\ Security\ Advisory\ (.*)\s\s+/) {
				$id = $1;
				$id =~ s/\s+$//;
			}
			$adv = join("", $adv,$line) ;#if ($id ne "");
			next LINE;
		}
		if ($adv && $id ) {
			$newAdv{$id} = $adv;
			#printf "\n\nNew Advisory:\n%s", $newAdv{$id};
			$adv="";
			$id="";
		}
	}
	if ($adv && $id ) {
		$newAdv{$id} = $adv;
		#printf "\n\nNew Advisory:\n%s", $newAdv{$id};
		$adv="";
		$id="";
	}
	
	return \%newAdv;
}

# read USN from stdin, assuming mbox format (separated by "^From ")
sub readUSN {
	my $adv="";
	my $id="";
	my %newAdv;
	
	msg 15, "Reading in Ubuntu Security Notice..";
	
	my @lines = <STDIN>;
	LINE: foreach my $line (@lines) {
		unless ($line =~ m/^From\ /) {
			if ($line =~ /^Ubuntu\ Security\ Notice\ (.*)\s\s+/) {
				$id=$1;
				$id =~ s/\s+$//;
			}
			$adv = join("", $adv,$line) ;#if ($id ne "");
			next LINE;
		}
		if ($adv && $id) {
			$newAdv{$id} = $adv;
			$adv="";
			$id="";
			#printf "\n\nNew Advisory:\n%s", $newAdv{$id};
		}
	}
	if ($adv && $id ) {
		$newAdv{$id} = $adv;
		#printf "\n\nNew Advisory:\n%s", $newAdv{$id};
		$adv="";
		$id="";
	}
	
	return \%newAdv;
}

## Should this advisory be skipped?
sub blacklistedAdvisory {
	my $dsa_id = shift or die "DSAquirks: no id given..";
	
	## check if DSA is blacklisted..
	my @id_blacklist = ("DSA-1975-1");
	
	grep ($_ eq $dsa_id, @id_blacklist) and return "true";
	
	return; # FALSE
}

## read in new security advisories from STDIN
## one by one, slow slow slow.. :-/
sub aptsec_newadv {
	my $newAdv;
	my $dist;
	
	given ($state->{"vendor"}) {
		when ($_ eq "debian") { $newAdv = readDSA; }
		when ($_ eq "ubuntu") { $newAdv = readUSN; }
#		when ($_ eq "redhat") { ($id,$adv) = checkRHSA; }
		default               { die "Unsupported distribution $_"; }
	};
	
	ADV: foreach my $id (keys %$newAdv) {
		# skip this advisory?
		next ADV if (blacklistedAdvisory($id));
		
		# if not known, process advisory 
		if ($dsatable->{$id}) {
			print $state->{"vendor"} . " advisory $id already known.\n";
		} else {
			$dsatable->{$id} = $newAdv->{$id};
			updateCVETables($id);
		}
	}
	
	# recompute all pkg statistics
	foreach my $srcpkg (keys %$src2dsa) {
		#print "srcpkg: $srcpkg\n";
		processCVEs($srcpkg);
	}
}

my $action;
$action = shift or $action = "help";

load_state();
detect_distribution();

given ($action) {

	when ($_ eq "update") {
		load_DBs;
		aptsec_update();
		save_DBs;
	}
	when ($_ eq "newadv")   {
		load_DBs; #or die "Failed to load some cached file(s), please rebuild with 'apt-sec update'\n";
		#$state->{"vendor"} = "ubuntu";
		aptsec_newadv();
		save_DBs;
	}
	when ($_ eq "system" || $_ eq "status") { 
		load_DBs or die "Failed to load some cached file(s), please rebuild with 'apt-sec update'\n";
		aptsec_system();
	}
	when ($_ eq "show")   {
		load_DBs or die "Failed to load some cached file(s), please rebuild with 'apt-sec update'\n";
		aptsec_show(shift);
	}
	when ($_ eq "attest")   {
		load_DBs or die "Failed to load some cached file(s), please rebuild with 'apt-sec update'\n";
		aptsec_attest(shift);
	}
	default {
		aptsec_help();
	}
};

save_state();;