gen-pkg-lists.sh 1.8 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. #!/bin/bash
  2. ## Update Packages.bz2 and Sources.bz2 to include all meta-data from last 7 days
  3. ## Only collects data from the unstable distribution.
  4. ##
  5. ## Must aggregate current last 7 days before updating meta-data for today,
  6. ## otherwise the Packages meta-data will be one day ahead of the Sha1Sums archive
  7. dest="/srv/http/htdocs/apt-sec/mirror"
  8. mirrors="/srv/apt-mirror/mirror/"
  9. echo "Updating Package and Sources lists.."
  10. if [ "A$1" = "Ainit" ]; then
  11. find $dest -name 'Packages.bz2-[0-9]' -delete
  12. find $dest -name 'Sources.bz2-[0-9]' -delete
  13. fi
  14. ## Collect meta-data of last 7 days
  15. ##
  16. # collect Packages
  17. for i in $dest/Packages.bz2-*; do
  18. bzcat $i
  19. done > $dest/Packages
  20. # collect Sources
  21. for i in $dest/Sources.bz2-*; do
  22. bzcat $i
  23. done > $dest/Sources
  24. # compress for download
  25. bzip2 -f $dest/Packages
  26. bzip2 -f $dest/Sources
  27. # Get today's files from archive and save data for the upcoming week
  28. #
  29. tmp=$(mktemp)
  30. find $mirrors -name Packages.bz2 > $tmp
  31. sed -i s/'\/srv\/apt-mirror\/mirror\/'// $tmp
  32. sed -i s/'binary-amd64\/Packages.bz2'/'source\/Sources.bz2'/ $tmp
  33. sed -i s/^/'http:\/\/'/ $tmp
  34. wget -qP $mirrors -ri $tmp
  35. rm $tmp
  36. # Do some filtering to reduce memory requirements
  37. # TODO: We should parse this cruft here and simply provide SQL for apt-sec!
  38. #for i in $(find $mirrors -name Packages.bz2 |grep "/sid/"); do
  39. for i in $(find $mirrors -name Packages.bz2); do
  40. bzcat $i
  41. done |grep "^$\|^Package:\|^Provides:\|^Tag:\|^Filename:" \
  42. |bzip2 > $dest/Packages.bz2-$(date +%w)
  43. #for i in $(find $mirrors -name Sources.bz2 |grep "/sid\/"); do
  44. for i in $(find $mirrors -name Sources.bz2); do
  45. bzcat $i
  46. done |grep "^$\|^Package:\|^Binary:\|^ \|^Version\|^Maintainer:" \
  47. |grep -E -v '^\ [a-z,A-Z,0-9]{64}\ [0-9]+\ ' \
  48. |grep -E -v '^\ [a-z,A-Z,0-9]{32}\ [0-9]+\ ' \
  49. |grep -E -v '^\ [a-z,A-Z,0-9]{40}\ [0-9]+\ ' \
  50. |bzip2 > $dest/Sources.bz2-$(date +%w)