TrustMiner2/gen-sha1-db.sh

#!/bin/bash

## Iterate through complete package archive to generate a complete list of SHA1
## Uses some magic to directly generate sha1sums without extracting to disk
##
## Command parameter 'init' will build from scratch. Very slow.
##
## Bin packages may contain links. For simplicity we currently dereference
## these("tar -xhv"), so we have each file with its associated sha1 measurement.

work=/srv/http/htdocs/apt-sec/
dest=$work/mirror/Sha1Sums.db
log=$work/mirror/sha1db.log

apt=/srv/apt-mirror
archive=$apt/mirror
var=$apt/var
NEW=$apt/var/NEW
ALL=$apt/var/ALL
ALL=$apt/var/DEL


action="$1"

#which sqlite3 > /dev/null || echo 'Need sqlite3. Exit.' 2>&1
#which sqlite3 > /dev/null || exit

# some overrides
[ "AA${PKGS}" = "AA" ] || archive="$PKGS"
[ "AA${DEST}" = "AA" ] || dest="$DEST"

echo "Updating Sha1Sums.."
echo > $log

sha1pipe () {
	file="$1"
	pkg="$(basename $1)"
	tmp=$(mktemp -d)
	ar -t $file |grep -q data.tar.gz && ( ar p $file data.tar.gz | tar xzC $tmp || echo "Can't extract from $file" )
	ar -t $file |grep -q data.tar.bz2 && ( ar p $file data.tar.bz2 | tar xjC $tmp || echo "Can't extract from $file" )
	cd $tmp
	## stream of <shasum>::<package>::<filepath>
	find . -type f -exec sha1sum {} \+ \
		| sed -e "s/'/\\\'/" -e "s/^/insert into sha2pkg (hash, pkg, file) VALUES('/" -e "s/  \./', '$pkg', '/" -e "s/$/')\;/"
	cd /
	rm -rf $tmp
}


case $action in

"init")
	# delete and rebuild sha1 db
	echo "Reinit - deleting current SHA1 db.."
	rm $dest
	sqlite3 $dest 'create table sha2pkg (hash char(20), pkg text, file text)'
	sqlite3 $dest 'create index [IDX_hash] on sha2pkg (hash)'

	echo "Reinitializing SHA1 DB.."
 	(echo "begin;";
	for pkg in $(find $archive -type f -iname \*.deb); do
		echo "Adding $pkg to sha1 db.." >> $log
		sha1pipe $pkg
	done; echo "commit"; ) > $dest.sql
	#cat $dest.sql |sqlite3 $dest >> $log 2>&1
	;;
"update")
	# record removed packets, remove sha1sums of packets recorded *last week*

	# delete packages that were removed last week
	# (how long are packages kept in online repo?)
	del=$DEL-$(date +%w)
	touch $del
	for pkg in $(cat $del); do
		echo "Deleting outdated package $pkg from db.." >> $log
		echo "delete * from sha2pkg where pkg='$pkg';"
	done > $dest.sql
 #	| sqlite3 $dest >> $log 2>&1

	# identify newly removed packages, overwriting last week's DEL record
	for pkg in $(cat $ALL.old); do
		grep -q $pkg $ALL || echo $pkg >> $del
	done
	cp $ALL $ALL.old

	# add sha1 sums for new pkgs
 	(echo "begin;";
	for pkg in $(cat $NEW); do
	 	pkg=$(echo $pkg|sed 's/.*:\/\///');
		echo "Adding $pkg to sha1 db.." >> $log
		sha1pipe $archive/$pkg
	done; echo "commit"; )  > $dest.sql
	#| sqlite3 $dest >> $log 2>&1
	
	# make sure we don't parse those packages again
	# (shouldn't wait for next apt-mirror call)
	echo -n > $NEW
	;;
*)
	echo
	echo "Usage: $0 (init|update)" 2>&1
	echo
	exit
esac