TrustMiner2/gen-pkg-lists.sh

#!/bin/bash

## Update Packages.bz2 and Sources.bz2 to include all meta-data from last 7 days
## Only collects data from the unstable distribution.
##
## Must aggregate current last 7 days before updating meta-data for today,
## otherwise the Packages meta-data will be one day ahead of the Sha1Sums archive

dest="/srv/http/htdocs/apt-sec/mirror"
mirrors="/srv/apt-mirror/mirror/"

echo "Updating Package and Sources lists.."

if [ "A$1" = "Ainit" ]; then
	find $dest -name 'Packages.bz2-[0-9]' -delete
	find $dest -name 'Sources.bz2-[0-9]' -delete
fi

## Collect meta-data of last 7 days
##
# collect Packages
for i in $dest/Packages.bz2-*; do
	bzcat $i
done > $dest/Packages

# collect Sources
for i in $dest/Sources.bz2-*; do
	bzcat $i
done > $dest/Sources

# compress for download
bzip2 -f $dest/Packages
bzip2 -f $dest/Sources


# Get today's files from archive and save data for the upcoming week
#
tmp=$(mktemp)
find $mirrors -name Packages.bz2 > $tmp
sed -i s/'\/srv\/apt-mirror\/mirror\/'// $tmp
sed -i s/'binary-amd64\/Packages.bz2'/'source\/Sources.bz2'/ $tmp
sed -i s/^/'http:\/\/'/ $tmp
wget -qP $mirrors -ri $tmp
rm $tmp

# Do some filtering to reduce memory requirements
# TODO: We should parse this cruft here and simply provide SQL for apt-sec!
#for i in $(find $mirrors -name Packages.bz2 |grep "/sid/"); do
for i in $(find $mirrors -name Packages.bz2); do
	bzcat $i
done |grep "^$\|^Package:\|^Provides:\|^Tag:\|^Filename:" \
	|bzip2 > $dest/Packages.bz2-$(date +%w)

#for i in $(find $mirrors -name Sources.bz2 |grep "/sid\/"); do
for i in $(find $mirrors -name Sources.bz2); do
	bzcat $i
done |grep "^$\|^Package:\|^Binary:\|^ \|^Version\|^Maintainer:" \
	|grep -E -v '^\ [a-z,A-Z,0-9]{64}\ [0-9]+\ ' \
	|grep -E -v '^\ [a-z,A-Z,0-9]{32}\ [0-9]+\ ' \
	|grep -E -v '^\ [a-z,A-Z,0-9]{40}\ [0-9]+\ ' \
	|bzip2 > $dest/Sources.bz2-$(date +%w)