[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible_pool_buildinfos: merge suite script into the main one
Holger Levsen
gitlab at salsa.debian.org
Mon Mar 1 13:50:55 GMT 2021
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
7e9f02e8 by Frédéric Pierret (fepitre) at 2021-03-01T14:49:07+01:00
reproducible_pool_buildinfos: merge suite script into the main one
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
6b23fd57 by Holger Levsen at 2021-03-01T14:50:15+01:00
jenkins: declare pool_buildinfos_suites as zombie job we do not want to see again
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
5 changed files:
- bin/common-functions.sh
- bin/jenkins_node_wrapper.sh
- bin/reproducible_pool_buildinfos.sh
- − bin/reproducible_pool_buildinfos_suites.sh
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/common-functions.sh
=====================================
@@ -1,7 +1,7 @@
#!/bin/bash
# vim: set noexpandtab:
-# Copyright 2014-2020 Holger Levsen <holger at layer-acht.org>
+# Copyright 2014-2021 Holger Levsen <holger at layer-acht.org>
# © 2018 Mattia Rizzolo <mattia at mapreri.org>
# released under the GPLv2
@@ -220,7 +220,7 @@ jenkins_zombie_check() {
# and we dont know why and when that happens,
# so just report those zombies here.
#
- ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|reproducible_*(odc2a|odxu4c|opi2b)|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-8]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks' || true)"
+ ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|reproducible_*(odc2a|odxu4c|opi2b)|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-8]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites' || true)"
if [ ! -z "$ZOMBIES" ] ; then
DIRTY=true
figlet 'zombies!!!'
=====================================
bin/jenkins_node_wrapper.sh
=====================================
@@ -83,8 +83,6 @@ elif [ "$1" = "reproducible_debian_rebuilder_prototype" ] ; then
exec /srv/jenkins/bin/reproducible_debian_rebuilder_prototype.sh ; croak "Exec failed";
elif [ "$1" = "reproducible_pool_buildinfos" ] ; then
exec /srv/jenkins/bin/reproducible_pool_buildinfos.sh ; croak "Exec failed";
-elif [ "$1" = "reproducible_pool_buildinfos_suites" ] ; then
- exec /srv/jenkins/bin/reproducible_pool_buildinfos_suites.sh ; croak "Exec failed";
elif [ "$1" = "reproducible_html_builtin-pho" ] ; then
exec /srv/jenkins/bin/reproducible_html_builtin-pho.sh ; croak "Exec failed";
elif [ "$1" = "janitor-setup-worker" ] ; then
=====================================
bin/reproducible_pool_buildinfos.sh
=====================================
@@ -20,6 +20,7 @@ set -e
BASEPATH=~jenkins/userContent/reproducible/debian
FTPPATH=$BASEPATH/ftp-master.debian.org/buildinfo
POOLPATH=$BASEPATH/buildinfo-pool
+POOLFILE=$POOLPATH/buildinfo-pool.list
mkdir -p $POOLPATH
# just in case
@@ -142,7 +143,52 @@ loop_through_all(){
done
}
+get_localsuite() {
+ if [ "$SUITE" = "unstable" ] ; then
+ LOCALSUITE="sid"
+ else
+ LOCALSUITE=$SUITE
+ fi
+}
+
+convert_db_output_to_buildinfo() {
+ sed -i '/^$/d' "$1"
+ # We create unfiltered list of predicted buildinfo filenames
+ cat "$1" | tr -d ' ' | \
+ awk -F'|' -v ARCH="${ARCH}" '{IDX=substr($1,1,1) ; SOURCE=$1; VERSION=gensub(/^.*:/, "", "g", $2); print "/buildinfo-pool/"IDX"/"SOURCE"/"SOURCE"_"VERSION"_"ARCH".buildinfo"}' \
+ > "$2"
+}
+
+
+filter_buildinfos() {
+ # We filter predicted buildinfo filenames if they really exist.
+ # If not, we report them into a separate .nonexistent file.
+ awk -v buildinfolist="$POOLPATH/buildinfo-pool_${SUITE}_${ARCH}.list" \
+ 'FNR==NR { a[$1]; next } $1 in a {print > buildinfolist}' "$1" "$2"
+}
+
+query_db_buildinfos() {
+ # We handle the case where ARCH has multiple concatenated values
+ PARSED_ARCH=$(for arch in ${ARCH//-/ }; do echo -n \'"$arch"\'\,; done)
+ PARSED_ARCH=${PARSED_ARCH::-1}
+ psql --tuples-only buildinfo <<EOF > "raw_packages_${LOCALSUITE}_${ARCH}.list"
+SELECT DISTINCT
+ p.source,p.version
+FROM
+ binary_packages p
+WHERE
+ p.suite='$LOCALSUITE' AND p.arch in (${PARSED_ARCH})
+ORDER BY
+ p.source
+EOF
+ convert_db_output_to_buildinfo "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+ filter_buildinfos "$POOLFILE" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+}
+
+#
# main
+#
+
if [ -n "$1" ] && [ -z "$2" ] ; then
# only run manually: do all days
loop_through_all
@@ -209,3 +255,25 @@ else
rm $PROBLEMS
fi
+# This part allows to create a buildinfo-pool file per suite and arch.
+# It queries the database created thanks to https://salsa.debian.org/bremner/builtin-pho
+# and use the primary buildinfo-pool file generated by reproducible_pool_buildinfos.sh
+# for filtering the results.
+ARCHES="
+ all-amd64
+ all-arm64
+ all
+ amd64
+ arm64
+ "
+
+SUITES="bullseye unstable"
+LOCALSUITE=""
+
+for ARCH in $ARCHES ; do
+ for SUITE in $SUITES ; do
+ get_localsuite
+ query_db_buildinfos
+ rm "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
+ done
+done
=====================================
bin/reproducible_pool_buildinfos_suites.sh deleted
=====================================
@@ -1,84 +0,0 @@
-#!/bin/bash
-# vim: set noexpandtab:
-
-# Copyright 2021 Frédéric Pierret <frederic.pierret at qubes-os.org>
-# released under the GPLv2
-
-# This script allows to create a buildinfo-pool file per suite and arch.
-# It queries the database created thanks to https://salsa.debian.org/bremner/builtin-pho
-# and use the primary buildinfo-pool file generated by reproducible_pool_buildinfos.sh
-# for filtering the results.
-
-set -e
-if [ "$DEBUG" == 1 ]; then
- set -x
-fi
-
-BASEPATH=~jenkins/userContent/reproducible/debian
-POOLFILE=$BASEPATH/buildinfo-pool.list
-
-get_localsuite() {
- if [ "$SUITE" = "unstable" ] ; then
- LOCALSUITE="sid"
- else
- LOCALSUITE=$SUITE
- fi
-}
-
-convert_db_output_to_buildinfo() {
- sed -i '/^$/d' "$1"
- # We create unfiltered list of predicted buildinfo filenames
- cat "$1" | tr -d ' ' | \
- awk -F'|' -v ARCH="${ARCH}" '{IDX=substr($1,1,1) ; SOURCE=$1; VERSION=gensub(/^.*:/, "", "g", $2); print "/buildinfo-pool/"IDX"/"SOURCE"/"SOURCE"_"VERSION"_"ARCH".buildinfo"}' \
- > "$2"
-}
-
-
-filter_buildinfos() {
- # We filter predicted buildinfo filenames if they really exist.
- # If not, we report them into a separate .nonexistent file.
- awk -v buildinfolist="buildinfo-pool_${SUITE}_${ARCH}.list" \
- 'FNR==NR { a[$1]; next } $1 in a {print > buildinfolist}' "$1" "$2"
-}
-
-query_db_buildinfos() {
- # We handle the case where ARCH has multiple concatenated values
- PARSED_ARCH=$(for arch in ${ARCH//-/ }; do echo -n \'"$arch"\'\,; done)
- PARSED_ARCH=${PARSED_ARCH::-1}
- psql --tuples-only buildinfo <<EOF > "raw_packages_${LOCALSUITE}_${ARCH}.list"
-SELECT DISTINCT
- p.source,p.version
-FROM
- binary_packages p
-WHERE
- p.suite='$LOCALSUITE' AND p.arch in (${PARSED_ARCH})
-ORDER BY
- p.source
-EOF
- convert_db_output_to_buildinfo "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
- filter_buildinfos "$POOLFILE" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
-}
-
-
-#
-# main
-#
-
-ARCHES="
- all-amd64
- all-arm64
- all
- amd64
- arm64
- "
-
-SUITES="bullseye unstable"
-LOCALSUITE=""
-
-for ARCH in $ARCHES ; do
- for SUITE in $SUITES ; do
- get_localsuite
- query_db_buildinfos
- rm "raw_packages_${LOCALSUITE}_${ARCH}.list" "raw_buildinfo-pool_${LOCALSUITE}_${ARCH}.list"
- done
-done
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -812,10 +812,6 @@
my_description: 'Link files for buildinfos.debian.net from date to pool structure'
my_hname: 'ionos7'
my_timed: '10 */2 * * *'
- - 'pool_buildinfos_suites':
- my_description: 'Create a buildinfo-pool file per suite and arch'
- my_hname: 'ionos7'
- my_timed: '40 */2 * * *'
- 'debian_rebuilder_prototype':
my_description: 'Prototype rebuilder for Debian'
my_hname: 'ionos7'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/ff2462cb5fc5d932d1d230a61b4af45442f4ffc9...6b23fd574ddf0f63e9ab27898723c424646dfa90
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/ff2462cb5fc5d932d1d230a61b4af45442f4ffc9...6b23fd574ddf0f63e9ab27898723c424646dfa90
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20210301/f0606d56/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list