[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 3 commits: buildinfos-pool: add per suite list

[Holger Levsen]

Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
7020347f by Frédéric Pierret (fepitre) at 2021-03-01T00:41:15+01:00
buildinfos-pool: add per suite list

Take into account @holger's comments

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
82e34937 by Frédéric Pierret (fepitre) at 2021-03-01T00:41:20+01:00
buildinfos-pool: reduce ARCHES list

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
4750f260 by Frédéric Pierret (fepitre) at 2021-03-01T00:41:24+01:00
reproducible: add buildinfos_suites job

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


3 changed files:

- bin/jenkins_node_wrapper.sh
- + bin/reproducible_pool_buildinfos_suites.sh
- job-cfg/reproducible.yaml


Changes:

=====================================
bin/jenkins_node_wrapper.sh
=====================================
@@ -83,6 +83,8 @@ elif [ "$1" = "reproducible_debian_rebuilder_prototype" ] ; then
 	exec /srv/jenkins/bin/reproducible_debian_rebuilder_prototype.sh ; croak "Exec failed";
 elif [ "$1" = "reproducible_pool_buildinfos" ] ; then
 	exec /srv/jenkins/bin/reproducible_pool_buildinfos.sh ; croak "Exec failed";
+elif [ "$1" = "reproducible_pool_buildinfos_suites" ] ; then
+	exec /srv/jenkins/bin/reproducible_pool_buildinfos_suites.sh ; croak "Exec failed";
 elif [ "$1" = "reproducible_html_builtin-pho" ] ; then
 	exec /srv/jenkins/bin/reproducible_html_builtin-pho.sh ; croak "Exec failed";
 elif [ "$1" = "janitor-setup-worker" ] ; then


=====================================
bin/reproducible_pool_buildinfos_suites.sh
=====================================
@@ -0,0 +1,85 @@
+#!/bin/bash
+# vim: set noexpandtab:
+
+# Copyright 2021 Frédéric Pierret <frederic.pierret at qubes-os.org>
+# released under the GPLv2
+
+# This script allows to create a buildinfo-pool file per suite and arch.
+# It queries the database created thanks to https://salsa.debian.org/bremner/builtin-pho
+# and use the primary buildinfo-pool file generated by reproducible_pool_buildinfos.sh
+# for filtering the results.
+
+set -e
+if [ "$DEBUG" == 1 ]; then
+	set -x
+fi
+
+BASEPATH=~jenkins/userContent/reproducible/debian
+FTPPATH=$BASEPATH/ftp-master.debian.org/buildinfo
+POOLFILE=$FTPPATH/buildinfo-pool.list
+
+get_localsuite() {
+	if [ "$SUITE" = "unstable" ] ; then
+		LOCALSUITE="sid"
+	else
+		LOCALSUITE=$SUITE
+	fi
+}
+
+convert_db_output_to_buildinfo() {
+	sed -i '/^$/d' "$1"
+	# We create unfiltered list of predicted buildinfo filenames
+	cat "$1" | tr -d ' ' | \
+		gawk -F'|' -v ARCH="${ARCH}" '{IDX=substr($1,1,1) ; SOURCE=$1; VERSION=gensub(/^.*:/, "", "g", $2); print "/buildinfo-pool/"IDX"/"SOURCE"/"SOURCE"_"VERSION"_"ARCH".buildinfo"}' \
+		> "$2"
+}
+
+
+filter_buildinfos() {
+	# We filter predicted buildinfo filenames if they really exist.
+	# If not, we report them into a separate .nonexistent file.
+	gawk -v buildinfolist="buildinfo-pool_${SUITE}_${ARCH}.list" \
+		'FNR==NR { a[$1]; next } { print > ($1 in a ? buildinfolist") }' "$1" "$2"
+}
+
+query_db_buildinfos() {
+	# We handle the case where ARCH has multiple concatenated values
+	PARSED_ARCH=$(for arch in ${ARCH//-/ }; do echo -n \'"$arch"\'\,; done)
+	PARSED_ARCH=${PARSED_ARCH::-1}
+	psql --tuples-only buildinfo <<EOF > "raw_packages_${LOCALSUITE}_${ARCH}.list"
+SELECT DISTINCT
+	p.source,p.version
+FROM
+	binary_packages p
+WHERE
+	p.suite='$LOCALSUITE' AND p.arch in (${PARSED_ARCH})
+ORDER BY
+	p.source
+EOF
+	convert_db_output_to_buildinfo "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+	filter_buildinfos "$POOLFILE" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+}
+
+
+#
+# main
+#
+
+ARCHES="
+	all-amd64
+	all-arm64
+	all
+	amd64
+	arm64
+	"
+
+SUITES="bullseye unstable"
+LOCALSUITE=""
+
+for ARCH in $ARCHES ; do
+	for SUITE in $SUITES ; do
+		get_localsuite
+		query_db_buildinfos
+		rm "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+	done
+done


=====================================
job-cfg/reproducible.yaml
=====================================
@@ -812,6 +812,10 @@
                     my_description: 'Link files for buildinfos.debian.net from date to pool structure'
                     my_hname: 'ionos7'
                     my_timed: '10 */2 * * *'
+                - 'pool_buildinfos_suites':
+                    my_description: 'Create a buildinfo-pool file per suite and arch'
+                    my_hname: 'ionos7'
+                    my_timed: '40 */2 * * *'
                 - 'debian_rebuilder_prototype':
                     my_description: 'Prototype rebuilder for Debian'
                     my_hname: 'ionos7'



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/625248922ad8b12564ee3044c42da9adb7c98e9f...4750f26035c9f3b3e39b07dcd2a71b063cba52f7

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/625248922ad8b12564ee3044c42da9adb7c98e9f...4750f26035c9f3b3e39b07dcd2a71b063cba52f7
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20210228/9d835791/attachment-0001.htm>