TrustMiner2/ubuntu-security-advisory.pl

#!/usr/bin/perl

###############################################################################
##
## Functions for downloading and parsing Ubuntu Security Advisories (USNs)
##
###############################################################################

use strict;
use warnings;

#use Storable; # persistant storage
use LWP::UserAgent;
#use Config::Auto; # libconfig-auto-perl
use Time::ParseDate; # libtime-modules-perl
#use IO::Uncompress::Bunzip2 qw(bunzip2 $Bunzip2Error);
#use IO::Uncompress::Gunzip qw(gunzip $GunzipError);
#use Digest::MD5 qw(md5_hex);
use POSIX qw(mktime);
#use File::Path qw(make_path);
use HTML::Parser;

use feature "switch";

## Fetch USN from ubuntu archive. Can't use tracker since dates are missing.
sub fetchUSN {
	my $id= shift;
	my $usn_base_url = shift;
	my $usn;
	
	trace "  Fetching USN-$id\n";
	
	my $url=	$usn_base_url . "usn-" . $id;
	debug "USN URL: $url\n";
	$usn = get $url;
	unless (defined $usn) {
		note "  USN-$id is not available\n";
		return "";
	}
	return $usn;
}

## Try to find new USNs by iteration, return table of USNs to process
## Ignores advisory revisions (not clear what to do with them, mostly junk)
sub checkUSNs($$) {
	my $state = shift;
	my $config = shift;
	my %usntable;
	my $next_usn = $state->{"next_adv"};
	my $rev = 1;
	my $id;
	
	trace "Checking for new USNs..\n";
 	
	if ($next_usn < $config->{"first_usn"}) {
		note "Cache was deleted, starting at USN $next_usn\n";
		$next_usn = $config->{"first_usn"};
	}

	$rev = 0 if ($next_usn == 60); # fuck
	$id	= sprintf("%d-%d",$next_usn, $rev);
	my $usn = fetchUSN($id, $config->{"usn_base_url"});
	
	until ($usn eq "") {
		debug "  Got USN-$id\n";
		
		# strip and save USN content
		$usn =~ s/.*<div\ class="usn">//si;
		$usn =~ s/<\/div>.*//si;
		$usntable{$id} = $usn;
		
		### check for advisory revisions?
		#until ($usn eq "") {
		#	msg 10, "  Got USN-$id";
		# $usn =~ s/.*<div\ class="usn">//si;
		# $usn =~ s/<\/div>.*//si;
		#	$usntable{$id} = $usn;
		#	
		#	$rev++;
		#	$rev++ unless (fixUSNquirks($next_usn,$rev)); # skip USN?
		#	$id = sprintf("%d-%d", $next_usn, $rev);
		#	$usn = fetchUSN($id);
		#}
		$rev = 1;
		$next_usn++;
		$next_usn++ if (blacklistedUSN("USN-".$next_usn));
		
		$rev = 0 if ($next_usn == 60);
		$id = sprintf("%d-%d", $next_usn, $rev);
		$usn = fetchUSN($id, $config->{"usn_base_url"});
	}

	return \%usntable;
}

## static map to correct errors in USNs
## returns fixed list of CVE IDs or 0 to skip DSA
sub fixUSNquirks {
	my $usn_id = shift;
	my $usn_state = shift;
	
	my @new_names = @{@$usn_state[0]};
	my $new_date = @$usn_state[1];
	my @new_cves = @{@$usn_state[2]};
	
	## These DSAs are totally screwed up..
	given ($usn_id) {
		when ($_ eq "291") {
		 	@new_cves = ("CVE-2006-0747", "CVE-2006-1861", "CVE-2006-2661");
		}
		when ($_ eq "853") {
		 	@new_cves = (
				"CVE-2009-0689", "CVE-2009-3274", "CVE-2009-3370", "CVE-2009-3371",
				"CVE-2009-3372", "CVE-2009-3373", "CVE-2009-3374", "CVE-2009-3375",
				"CVE-2009-3376", "CVE-2009-3377", "CVE-2009-3380", "CVE-2009-3381",
				"CVE-2009-3382", "CVE-2009-3383");
		}
	};
	
	return (\@new_names, $new_date, \@new_cves);
}

## Should this advisory be skipped?
sub blacklistedUSN {
	my $id = shift or die "Advisory blacklist: no id given..";
	
	## check if Adv is blacklisted..
	my @id_blacklist = ("USN-667", "USN-718", "USN-910", "USN-935",
		"USN-950", "USN-1034", "USN-1049", "USN-1050");
	
	grep ($_ eq $id, @id_blacklist) and return "true";
	
	return; # FALSE
}


## Parse USN data and return array
## (src-pkg-name date (CVE-id)*)
sub parseUSNhtml {
	my $usn = shift;
	
	my @usn_names;
	my @usn_CVEs;
	my $usn_date;
	my $usn_type;
	my @tmp;
	
	my @lines = split(/\n/,$usn);
  LINE: foreach my $line (@lines) {

    # Date Reported -> $dsa_date
    unless ($usn_date) {
      #print "Looking for date\n";
      #print "Line: $line\n";
      if ($line =~ /^Ubuntu\ Security\ Notice\ .*\d+-\d+\s+(\w+)\s+(\d+),\s+(\d+)/) {
        $usn_date = parsedate("$1 $2, $3");
      }
      next LINE;
    }

    # Affected Packages -> @dsa_names
    unless (@usn_names) {
      #print "Looking for name\n";
      #print "Line: $line\n";
      if ($line =~ m/^CVE-/ || $line =~ m/^CAN-/ || $line =~ m/^http.*launchpad.net\/bugs/|| $line =~ m/^===========/) {
        @usn_names = @tmp;
        @tmp=();
        $usn_type = pop @usn_names;
      } else {
        foreach my $w (split(/\s+/, $line)) {
          $w =~ s/,\s*//;
          push @tmp, $w;
        }
        next LINE;
      }
    }

    # Security database references (CVEs) -> @dsa_CVEs
    unless (@usn_CVEs) {
      #print "Looking for cve\n";
      #print "Line: $line\n";
      if ($line =~ m/^===========/) {
        @usn_CVEs = @tmp;
      } else {
        foreach my $w (split(/\s+/, $line)) {
          $w =~ s/,\s*//;
          push @tmp, $w;
        }
      }
    }
	}
	
	return (\@usn_names, $usn_date, \@usn_CVEs);
}


1;