[Git][security-tracker-team/security-tracker][master] Generalize the code to fetch Packages/Sources updates
Raphaël Hertzog
hertzog at debian.org
Thu May 31 12:57:41 BST 2018
Raphaël Hertzog pushed to branch master at Debian Security Tracker / security-tracker
Commits:
5189dc09 by Raphaël Hertzog at 2018-05-31T13:54:35+02:00
Generalize the code to fetch Packages/Sources updates
It's now easy to extend by adding a supplementary lib/foobar-releases.mk
file.
I verified that the set of downloaded files are identical with the
exception of a bug that got fixed in the process: the update-backports-%
rule used to download files for all sections (main, contrib, non-free)
but saved them all under the name corresponding to the main section.
- - - - -
2 changed files:
- Makefile
- + lib/debian-releases.mk
Changes:
=====================================
Makefile
=====================================
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,39 @@
PYTHON = python
PYTHON_MODULES = $(wildcard lib/python/*.py)
-BUG_LISTS = $(wildcard data/*/list)
-# Adjust these if necessary. The architecture selection is rather
-# arbitrary at the moment. More architectures can be added later.
+# The following variables need to be kept up-to-date and can be adjusted
+OLDOLDSTABLE = wheezy
+OLDSTABLE = jessie
+STABLE = stretch
+TESTING = buster
+
+MIRROR = http://debian.csail.mit.edu/debian
+SECURITY_MIRROR = http://security.debian.org/debian-security
-MIRROR = http://debian.csail.mit.edu/debian/
wheezy_ARCHS = amd64 armel armhf i386
jessie_ARCHS = amd64 arm64 armel armhf i386 mips mipsel powerpc ppc64el s390x
stretch_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x
buster_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x
sid_ARCHS = amd64 arm64 armel armhf hurd-i386 i386 kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x
-OLDOLDSTABLE = wheezy
-OLDSTABLE = jessie
-STABLE = stretch
-TESTING = buster
+# The rest of the file should not need to be edited
+
+# Include the definitions of the releases to be fetched
+include lib/*-releases.mk
+
+# There's a "RELEASES" variable defining the name of all releases to
+# process. Then for each release, there are 5 associated variables:
+# <name>_MIRROR: the base URL of the mirror hosting the repository
+# <name>_DIST: the sub-directory in $MIRROR/dists so that
+# $MIRROR/dists/$DIST/Release is a valid URL
+# <name>_ARCHS: the list of architectures supported in this release
+# <name>_RELEASE: the release name for the security tracker
+# <name>_SUBRELEASE: the sub-release identifier for the security tracker
+
+Q=@
+ifneq ($(V),)
+Q=
+endif
all:
$(PYTHON) bin/update-db data/security.db
@@ -51,157 +69,33 @@ stamps/DLA-syntax: data/DLA/list bin/check-syntax $(PYTHON_MODULES)
serve:
@bash bin/test-web-server
-.PHONY: update-packages update-testing update-stable update-oldstable update-oldoldstable
-update-packages: update-testing
- set -e ; for rel in sid ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/source/Sources \
- data/packages/$${rel}__$${archive}_Sources ; \
- done ; \
- for arch in $(sid_ARCHS) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
- data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-testing:
- set -e ; for rel in $(TESTING) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/source/Sources \
- data/packages/$${rel}__$${archive}_Sources ; \
- done ; \
- for arch in $($(TESTING)_ARCHS) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
- data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-stable:
- set -e ; for rel in $(STABLE) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/source/Sources \
- data/packages/$${rel}__$${archive}_Sources ; \
- done ; \
- for arch in $($(STABLE)_ARCHS) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
- data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-oldstable:
- set -e ; for rel in $(OLDSTABLE) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/source/Sources \
- data/packages/$${rel}__$${archive}_Sources ; \
- done ; \
- for arch in $($(OLDSTABLE)_ARCHS) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
- data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-oldoldstable:
- set -e ; for rel in $(OLDOLDSTABLE) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/source/Sources \
- data/packages/$${rel}__$${archive}_Sources ; \
- done ; \
- for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \
- for archive in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \
- data/packages/$${rel}__$${archive}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-.PHONY: update-security update-old-security update-oldold-security update-testing-security
-ST_MIRROR = http://security.debian.org/dists/$(TESTING)/updates
-ST_FILE = data/packages/$(TESTING)_security_
-update-testing-security:
+.PHONY: update-packages
+update-packages: $(foreach release,$(RELEASES),update-$(release))
+
+# This rule is a bit complicated as we need to escape $ for the shell twice,
+# once for the eval and once for the usual make processing
+define add_update_rule =
+.PHONY: update-$(1)
+update-$(1):
+ $$(Q)set -e; \
+ prefix="$$($(1)_RELEASE)_$$($(1)_SUBRELEASE)"; \
+ dist="$$($(1)_DIST)"; \
+ mirror="$$($(1)_MIRROR)"; \
for section in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(ST_MIRROR)/$$section/source/Sources $(ST_FILE)$${section}_Sources ; \
- set -e ; for arch in $($(TESTING)_ARCHS) ; do \
- $(PYTHON) bin/apt-update-file \
- $(ST_MIRROR)/$$section/binary-$${arch}/Packages $(ST_FILE)$${section}_$${arch}_Packages ; \
- done ; \
- done
-
-SEC_MIRROR = http://security.debian.org/dists
-update-security: update-old-security update-oldold-security
- for archive in $(STABLE); do \
- for section in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \
- data/packages/$${archive}_security_$${section}_Sources ; \
- for arch in $($(STABLE)_ARCHS) ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \
- data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-old-security:
- for archive in $(OLDSTABLE); do \
- for section in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \
- data/packages/$${archive}_security_$${section}_Sources ; \
- for arch in $($(OLDSTABLE)_ARCHS) ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \
- data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \
- done ; \
- done ; \
- done
-
-update-oldold-security:
- for archive in $(OLDOLDSTABLE); do \
- for section in main contrib non-free ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \
- data/packages/$${archive}_security_$${section}_Sources ; \
- for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \
- $(PYTHON) bin/apt-update-file \
- $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \
- data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \
- done ; \
- done ; \
+ echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/source/Sources"; \
+ $$(PYTHON) bin/apt-update-file \
+ $$$$mirror/dists/$$$$dist/$$$$section/source/Sources \
+ data/packages/$$$${prefix}_$$$${section}_Sources ; \
+ for arch in $$($(1)_ARCHS) ; do \
+ echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages"; \
+ $$(PYTHON) bin/apt-update-file \
+ $$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages \
+ data/packages/$$$${prefix}_$$$${section}_$$$${arch}_Packages ; \
+ done; \
done
-update-backports: update-backports-$(STABLE) update-backports-$(OLDSTABLE) update-backports-$(OLDOLDSTABLE)
-
-update-backports-%:
- set -e && archive=$(shell echo $@ | cut -d- -f3) ; \
- for section in main contrib non-free ; do \
- for arch in $($(shell echo $@ | cut -d- -f3)_ARCHS) ; do \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$${archive}-backports/$$section/binary-$$arch/Packages \
- data/packages/$${archive}-backports__main_$${arch}_Packages ; \
- done ; \
- $(PYTHON) bin/apt-update-file \
- $(MIRROR)/dists/$${archive}-backports/$$section/source/Sources \
- data/packages/$${archive}-backports__main_Sources ; \
- done ; \
+endef
+$(foreach release,$(RELEASES),$(eval $(call add_update_rule,$(release))))
update-lists:
git fetch -q origin && git checkout -f origin/master -- data
@@ -228,4 +122,4 @@ update-compare-nvd:
done
bin/compare-nvd-cve 2> compare-nvd-cve.log
-update-all: update-nvd update-lists update-packages update-oldoldstable update-oldstable update-stable update-security update-testing-security update-packages update-backports all
+update-all: update-nvd update-lists update-packages all
=====================================
lib/debian-releases.mk
=====================================
--- /dev/null
+++ b/lib/debian-releases.mk
@@ -0,0 +1,39 @@
+# This file defines the variables describing all Debian repositories
+# that need to be fetched in the "update-packages" process
+
+BACKPORT_RELEASES := $(OLDOLDSTABLE) $(OLDSTABLE) $(STABLE)
+SECURITY_RELEASES := $(BACKPORT_RELEASES) $(TESTING)
+MAIN_RELEASES := $(SECURITY_RELEASES) sid
+
+# Define the variables for the release on the main mirror
+define add_main_release =
+$(1)_MIRROR = $$(MIRROR)
+$(1)_DIST = $(1)
+$(1)_ARCHS ?= amd64 arm64 armel armhf hurd-i386 i386 kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x
+$(1)_RELEASE = $(1)
+$(1)_SUBRELEASE =
+RELEASES += $(1)
+endef
+$(foreach release,$(MAIN_RELEASES),$(eval $(call add_main_release,$(release))))
+
+# Define the variables for the releases on security.debian.org
+define add_security_release =
+$(1)_security_MIRROR = $$(SECURITY_MIRROR)
+$(1)_security_DIST = $(1)/updates
+$(1)_security_ARCHS = $$($(1)_ARCHS)
+$(1)_security_RELEASE = $(1)
+$(1)_security_SUBRELEASE = security
+RELEASES += $(1)_security
+endef
+$(foreach release,$(SECURITY_RELEASES),$(eval $(call add_security_release,$(release))))
+
+# Define the variables for the *-backports releases
+define add_backport_release =
+$(1)_backports_MIRROR = $$(MIRROR)
+$(1)_backports_DIST = $(1)-backports
+$(1)_backports_ARCHS = $$($(1)_ARCHS)
+$(1)_backports_RELEASE = $(1)-backports
+$(1)_backports_SUBRELEASE =
+RELEASES += $(1)_backports
+endef
+$(foreach release,$(BACKPORT_RELEASES),$(eval $(call add_backport_release,$(release))))
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/commit/5189dc097b5bfe246c683102b6b4caabaefe00f0
--
View it on GitLab: https://salsa.debian.org/security-tracker-team/security-tracker/commit/5189dc097b5bfe246c683102b6b4caabaefe00f0
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-security-tracker-commits/attachments/20180531/a9613cb2/attachment-0001.html>
More information about the debian-security-tracker-commits
mailing list