[Git][qa/jenkins.debian.net][master] Move the list of workers into a separate file, so it's accessible to both scripts
Mattia Rizzolo (@mattia)
gitlab at salsa.debian.org
Sat Mar 9 17:03:12 GMT 2024
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
fea359c6 by Mattia Rizzolo at 2024-03-09T18:00:44+01:00
Move the list of workers into a separate file, so it's accessible to both scripts
Also, this is a plain file regularly re-read, not sourced, it's possible
to spin up/down workers without actually starting/sopping processes.
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
3 changed files:
- bin/reproducible_build_service.sh
- + bin/reproducible_build_workers
- bin/reproducible_worker.sh
Changes:
=====================================
bin/reproducible_build_service.sh
=====================================
@@ -35,6 +35,14 @@ startup_workers() {
# continue loop if the worker to be started is already running
#
WORKER_NAME=${ARCH}_$i
+
+
+ # shellcheck disable=SC2143
+ if [ -z "$(grep "$WORKER_NAME " reproducible_build_workers|grep -v '^#')" ]; then
+ continue
+ fi
+
+
#SERVICE_NAME="reproducible_build@${WORKER_NAME}.service"
WORKER_BIN=/srv/jenkins/bin/reproducible_worker.sh
=====================================
bin/reproducible_build_workers
=====================================
@@ -0,0 +1,129 @@
+# worker name node 1 node 2
+
+
+# amd64, i386 and arm64 nodes are chosen in a way that one build always runs
+# on a node running in the future, the other on a node with correct date.
+# armhf builds are distributed by the build capacity of the nodes, see below.
+
+
+amd64_1 ionos1-amd64 ionos5-amd64
+amd64_2 ionos5-amd64 ionos1-amd64
+amd64_3 ionos1-amd64 ionos15-amd64
+amd64_4 ionos15-amd64 ionos1-amd64
+amd64_5 ionos11-amd64 ionos5-amd64
+amd64_6 ionos5-amd64 ionos11-amd64
+amd64_7 ionos11-amd64 ionos15-amd64
+amd64_8 ionos15-amd64 ionos11-amd64
+amd64_9 ionos1-amd64 ionos5-amd64
+amd64_10 ionos5-amd64 ionos1-amd64
+amd64_11 ionos1-amd64 ionos15-amd64
+amd64_12 ionos15-amd64 ionos1-amd64
+amd64_13 ionos11-amd64 ionos5-amd64
+amd64_14 ionos5-amd64 ionos11-amd64
+amd64_15 ionos11-amd64 ionos15-amd64
+amd64_16 ionos15-amd64 ionos11-amd64
+amd64_17 ionos1-amd64 ionos5-amd64
+amd64_18 ionos5-amd64 ionos1-amd64
+amd64_19 ionos1-amd64 ionos15-amd64
+amd64_20 ionos15-amd64 ionos1-amd64
+amd64_21 ionos11-amd64 ionos5-amd64
+amd64_22 ionos5-amd64 ionos11-amd64
+amd64_23 ionos11-amd64 ionos15-amd64
+amd64_24 ionos15-amd64 ionos11-amd64
+amd64_25 ionos1-amd64 ionos5-amd64
+amd64_26 ionos5-amd64 ionos1-amd64
+amd64_27 ionos11-amd64 ionos15-amd64
+amd64_28 ionos15-amd64 ionos11-amd64
+#amd64_29 ionos1-amd64 ionos15-amd64
+#amd64_30 ionos15-amd64 ionos1-amd64
+#amd64_31 ionos11-amd64 ionos5-amd64
+#amd64_32 ionos5-amd64 ionos11-amd64
+
+
+# i386
+##i386_1 ionos2-i386 ionos6-i386
+##i386_2 ionos6-i386 ionos2-i386
+##i386_3 ionos2-i386 ionos16-i386
+##i386_4 ionos16-i386 ionos2-i386
+##i386_5 ionos12-i386 ionos6-i386
+##i386_6 ionos6-i386 ionos12-i386
+##i386_7 ionos12-i386 ionos16-i386
+##i386_8 ionos16-i386 ionos12-i386
+#i386_9 ionos2-i386 ionos6-i386
+#i386_10 ionos16-i386 ionos12-i386
+#i386_11 ionos6-i386 ionos2-i386
+#i386_12 ionos12-i386 ionos16-i386
+
+
+# arm64
+arm64_1 codethink01-arm64 codethink02-arm64
+#arm64_2 codethink01-arm64 codethink03-arm64
+#arm64_3 codethink01-arm64 codethink04-arm64
+arm64_4 codethink02-arm64 codethink01-arm64
+arm64_5 codethink03-arm64 codethink01-arm64
+arm64_6 codethink04-arm64 codethink01-arm64
+arm64_7 codethink02-arm64 codethink03-arm64
+#arm64_8 codethink02-arm64 codethink04-arm64
+arm64_9 codethink03-arm64 codethink02-arm64
+arm64_10 codethink04-arm64 codethink02-arm64
+arm64_11 codethink03-arm64 codethink04-arm64
+arm64_12 codethink04-arm64 codethink03-arm64
+arm64_13 codethink01-arm64 codethink02-arm64
+arm64_14 codethink01-arm64 codethink03-arm64
+arm64_15 codethink01-arm64 codethink04-arm64
+#arm64_16 codethink02-arm64 codethink01-arm64
+#arm64_17 codethink03-arm64 codethink01-arm64
+arm64_18 codethink04-arm64 codethink01-arm64
+arm64_19 codethink02-arm64 codethink03-arm64
+arm64_20 codethink02-arm64 codethink04-arm64
+#arm64_21 codethink03-arm64 codethink02-arm64
+#arm64_22 codethink04-arm64 codethink02-arm64
+arm64_23 codethink03-arm64 codethink04-arm64
+#arm64_24 codethink04-arm64 codethink03-arm64
+
+
+# to choose new armhf jobs:
+ # for i in cbxi4pro0 wbq0 ff64a cbxi4a cbxi4b ff4a virt32a virt32b virt32c virt32z virt64a virt64b virt64c virt64z ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf|nl ; done
+ # 6-8 jobs for quad-cores with 15 gb ram
+ # 6-7 jobs for quad-cores with 7 gb ram
+ # 6 jobs for quad-cores with 4 gb ram
+ # 4 jobs for quad-cores with 2gb of ram
+#
+# Don't forget to update README with the number of builders…!
+#
+##armhf_1 cbxi4a-armhf-rb virt64z-armhf-rb
+##armhf_2 virt32a-armhf-rb virt64c-armhf-rb
+##armhf_3 ff4a-armhf-rb virt64z-armhf-rb
+##armhf_4 virt64z-armhf-rb ff4a-armhf-rb
+##armhf_5 virt64a-armhf-rb cbxi4b-armhf-rb
+##armhf_6 virt64b-armhf-rb virt32z-armhf-rb
+##armhf_7 virt64b-armhf-rb cbxi4a-armhf-rb
+##armhf_8 virt64a-armhf-rb virt32c-armhf-rb
+##armhf_9 virt32a-armhf-rb virt64z-armhf-rb
+##armhf_10 wbq0-armhf-rb virt64a-armhf-rb
+##armhf_11 wbq0-armhf-rb virt64b-armhf-rb
+##armhf_12 virt64a-armhf-rb virt32b-armhf-rb
+##armhf_13 ff64a-armhf-rb virt32b-armhf-rb
+##armhf_14 virt32z-armhf-rb ff64a-armhf-rb
+##armhf_15 virt64b-armhf-rb virt32z-armhf-rb
+##armhf_16 virt32b-armhf-rb virt64z-armhf-rb
+##armhf_17 virt32b-armhf-rb virt64b-armhf-rb
+##armhf_18 ff64a-armhf-rb virt32a-armhf-rb
+##armhf_19 virt64c-armhf-rb cbxi4a-armhf-rb
+##armhf_20 virt64c-armhf-rb wbq0-armhf-rb
+##armhf_21 cbxi4a-armhf-rb ff64a-armhf-rb
+##armhf_22 cbxi4b-armhf-rb virt64c-armhf-rb
+##armhf_23 cbxi4b-armhf-rb ff64a-armhf-rb
+##armhf_24 ff4a-armhf-rb virt64b-armhf-rb
+##armhf_25 virt32c-armhf-rb virt64z-armhf-rb
+##armhf_26 virt64z-armhf-rb cbxi4b-armhf-rb
+##armhf_27 virt64z-armhf-rb virt32a-armhf-rb
+##armhf_28 virt64a-armhf-rb cbxi4pro0-armhf-rb
+##armhf_29 virt32z-armhf-rb virt64a-armhf-rb
+##armhf_30 ff64a-armhf-rb virt32c-armhf-rb
+##armhf_31 virt64c-armhf-rb ff4a-armhf-rb
+##armhf_32 virt64c-armhf-rb virt32c-armhf-rb
+##armhf_33 virt32z-armhf-rb virt64c-armhf-rb
+##armhf_34 virt32z-armhf-rb virt64a-armhf-rb
+##armhf_35 cbxi4pro0-armhf-rb virt64b-armhf-rb
+
=====================================
bin/reproducible_worker.sh
=====================================
@@ -28,131 +28,13 @@ common_init "$@"
# this function defines which builds take place on which nodes
#
choose_nodes() {
- case $1 in
- #
- # amd64, i386 and arm64 nodes are chosen in a way that one build always runs
- # on a node running in the future, the other on a node with correct date.
- # armhf builds are distributed by the build capacity of the nodes, see below.
- #
- amd64_1) NODE1=ionos1-amd64 NODE2=ionos5-amd64 ;;
- amd64_2) NODE1=ionos5-amd64 NODE2=ionos1-amd64 ;;
- amd64_3) NODE1=ionos1-amd64 NODE2=ionos15-amd64 ;;
- amd64_4) NODE1=ionos15-amd64 NODE2=ionos1-amd64 ;;
- amd64_5) NODE1=ionos11-amd64 NODE2=ionos5-amd64 ;;
- amd64_6) NODE1=ionos5-amd64 NODE2=ionos11-amd64 ;;
- amd64_7) NODE1=ionos11-amd64 NODE2=ionos15-amd64 ;;
- amd64_8) NODE1=ionos15-amd64 NODE2=ionos11-amd64 ;;
- amd64_9) NODE1=ionos1-amd64 NODE2=ionos5-amd64 ;;
- amd64_10) NODE1=ionos5-amd64 NODE2=ionos1-amd64 ;;
- amd64_11) NODE1=ionos1-amd64 NODE2=ionos15-amd64 ;;
- amd64_12) NODE1=ionos15-amd64 NODE2=ionos1-amd64 ;;
- amd64_13) NODE1=ionos11-amd64 NODE2=ionos5-amd64 ;;
- amd64_14) NODE1=ionos5-amd64 NODE2=ionos11-amd64 ;;
- amd64_15) NODE1=ionos11-amd64 NODE2=ionos15-amd64 ;;
- amd64_16) NODE1=ionos15-amd64 NODE2=ionos11-amd64 ;;
- amd64_17) NODE1=ionos1-amd64 NODE2=ionos5-amd64 ;;
- amd64_18) NODE1=ionos5-amd64 NODE2=ionos1-amd64 ;;
- amd64_19) NODE1=ionos1-amd64 NODE2=ionos15-amd64 ;;
- amd64_20) NODE1=ionos15-amd64 NODE2=ionos1-amd64 ;;
- amd64_21) NODE1=ionos11-amd64 NODE2=ionos5-amd64 ;;
- amd64_22) NODE1=ionos5-amd64 NODE2=ionos11-amd64 ;;
- amd64_23) NODE1=ionos11-amd64 NODE2=ionos15-amd64 ;;
- amd64_24) NODE1=ionos15-amd64 NODE2=ionos11-amd64 ;;
- amd64_25) NODE1=ionos1-amd64 NODE2=ionos5-amd64 ;;
- amd64_26) NODE1=ionos5-amd64 NODE2=ionos1-amd64 ;;
- amd64_27) NODE1=ionos11-amd64 NODE2=ionos15-amd64 ;;
- amd64_28) NODE1=ionos15-amd64 NODE2=ionos11-amd64 ;;
- #amd64_29) NODE1=ionos1-amd64 NODE2=ionos15-amd64 ;;
- #amd64_30) NODE1=ionos15-amd64 NODE2=ionos1-amd64 ;;
- #amd64_31) NODE1=ionos11-amd64 NODE2=ionos5-amd64 ;;
- #amd64_32) NODE1=ionos5-amd64 NODE2=ionos11-amd64 ;;
-
- # i386
- ##i386_1) NODE1=ionos2-i386 NODE2=ionos6-i386 ;;
- ##i386_2) NODE1=ionos6-i386 NODE2=ionos2-i386 ;;
- ##i386_3) NODE1=ionos2-i386 NODE2=ionos16-i386 ;;
- ##i386_4) NODE1=ionos16-i386 NODE2=ionos2-i386 ;;
- ##i386_5) NODE1=ionos12-i386 NODE2=ionos6-i386 ;;
- ##i386_6) NODE1=ionos6-i386 NODE2=ionos12-i386 ;;
- ##i386_7) NODE1=ionos12-i386 NODE2=ionos16-i386 ;;
- ##i386_8) NODE1=ionos16-i386 NODE2=ionos12-i386 ;;
- #i386_9) NODE1=ionos2-i386 NODE2=ionos6-i386 ;;
- #i386_10) NODE1=ionos16-i386 NODE2=ionos12-i386 ;;
- #i386_11) NODE1=ionos6-i386 NODE2=ionos2-i386 ;;
- #i386_12) NODE1=ionos12-i386 NODE2=ionos16-i386 ;;
-
- # arm64
- arm64_1) NODE1=codethink01-arm64 NODE2=codethink02-arm64 ;;
- #arm64_2) NODE1=codethink01-arm64 NODE2=codethink03-arm64 ;;
- #arm64_3) NODE1=codethink01-arm64 NODE2=codethink04-arm64 ;;
- arm64_4) NODE1=codethink02-arm64 NODE2=codethink01-arm64 ;;
- arm64_5) NODE1=codethink03-arm64 NODE2=codethink01-arm64 ;;
- arm64_6) NODE1=codethink04-arm64 NODE2=codethink01-arm64 ;;
- arm64_7) NODE1=codethink02-arm64 NODE2=codethink03-arm64 ;;
- #arm64_8) NODE1=codethink02-arm64 NODE2=codethink04-arm64 ;;
- arm64_9) NODE1=codethink03-arm64 NODE2=codethink02-arm64 ;;
- arm64_10) NODE1=codethink04-arm64 NODE2=codethink02-arm64 ;;
- arm64_11) NODE1=codethink03-arm64 NODE2=codethink04-arm64 ;;
- arm64_12) NODE1=codethink04-arm64 NODE2=codethink03-arm64 ;;
- arm64_13) NODE1=codethink01-arm64 NODE2=codethink02-arm64 ;;
- arm64_14) NODE1=codethink01-arm64 NODE2=codethink03-arm64 ;;
- arm64_15) NODE1=codethink01-arm64 NODE2=codethink04-arm64 ;;
- #arm64_16) NODE1=codethink02-arm64 NODE2=codethink01-arm64 ;;
- #arm64_17) NODE1=codethink03-arm64 NODE2=codethink01-arm64 ;;
- arm64_18) NODE1=codethink04-arm64 NODE2=codethink01-arm64 ;;
- arm64_19) NODE1=codethink02-arm64 NODE2=codethink03-arm64 ;;
- arm64_20) NODE1=codethink02-arm64 NODE2=codethink04-arm64 ;;
- #arm64_21) NODE1=codethink03-arm64 NODE2=codethink02-arm64 ;;
- #arm64_22) NODE1=codethink04-arm64 NODE2=codethink02-arm64 ;;
- arm64_23) NODE1=codethink03-arm64 NODE2=codethink04-arm64 ;;
- #arm64_24) NODE1=codethink04-arm64 NODE2=codethink03-arm64 ;;
- # to choose new armhf jobs:
- # for i in cbxi4pro0 wbq0 ff64a cbxi4a cbxi4b ff4a virt32a virt32b virt32c virt32z virt64a virt64b virt64c virt64z ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf|nl ; done
- # 6-8 jobs for quad-cores with 15 gb ram
- # 6-7 jobs for quad-cores with 7 gb ram
- # 6 jobs for quad-cores with 4 gb ram
- # 4 jobs for quad-cores with 2gb of ram
- #
- # Don't forget to update README with the number of builders…!
- #
- ##armhf_1) NODE1=cbxi4a-armhf-rb NODE2=virt64z-armhf-rb ;;
- ##armhf_2) NODE1=virt32a-armhf-rb NODE2=virt64c-armhf-rb ;;
- ##armhf_3) NODE1=ff4a-armhf-rb NODE2=virt64z-armhf-rb ;;
- ##armhf_4) NODE1=virt64z-armhf-rb NODE2=ff4a-armhf-rb ;;
- ##armhf_5) NODE1=virt64a-armhf-rb NODE2=cbxi4b-armhf-rb ;;
- ##armhf_6) NODE1=virt64b-armhf-rb NODE2=virt32z-armhf-rb ;;
- ##armhf_7) NODE1=virt64b-armhf-rb NODE2=cbxi4a-armhf-rb ;;
- ##armhf_8) NODE1=virt64a-armhf-rb NODE2=virt32c-armhf-rb ;;
- ##armhf_9) NODE1=virt32a-armhf-rb NODE2=virt64z-armhf-rb ;;
- ##armhf_10) NODE1=wbq0-armhf-rb NODE2=virt64a-armhf-rb ;;
- ##armhf_11) NODE1=wbq0-armhf-rb NODE2=virt64b-armhf-rb ;;
- ##armhf_12) NODE1=virt64a-armhf-rb NODE2=virt32b-armhf-rb ;;
- ##armhf_13) NODE1=ff64a-armhf-rb NODE2=virt32b-armhf-rb ;;
- ##armhf_14) NODE1=virt32z-armhf-rb NODE2=ff64a-armhf-rb ;;
- ##armhf_15) NODE1=virt64b-armhf-rb NODE2=virt32z-armhf-rb ;;
- ##armhf_16) NODE1=virt32b-armhf-rb NODE2=virt64z-armhf-rb ;;
- ##armhf_17) NODE1=virt32b-armhf-rb NODE2=virt64b-armhf-rb ;;
- ##armhf_18) NODE1=ff64a-armhf-rb NODE2=virt32a-armhf-rb ;;
- ##armhf_19) NODE1=virt64c-armhf-rb NODE2=cbxi4a-armhf-rb ;;
- ##armhf_20) NODE1=virt64c-armhf-rb NODE2=wbq0-armhf-rb ;;
- ##armhf_21) NODE1=cbxi4a-armhf-rb NODE2=ff64a-armhf-rb ;;
- ##armhf_22) NODE1=cbxi4b-armhf-rb NODE2=virt64c-armhf-rb ;;
- ##armhf_23) NODE1=cbxi4b-armhf-rb NODE2=ff64a-armhf-rb ;;
- ##armhf_24) NODE1=ff4a-armhf-rb NODE2=virt64b-armhf-rb ;;
- ##armhf_25) NODE1=virt32c-armhf-rb NODE2=virt64z-armhf-rb ;;
- ##armhf_26) NODE1=virt64z-armhf-rb NODE2=cbxi4b-armhf-rb ;;
- ##armhf_27) NODE1=virt64z-armhf-rb NODE2=virt32a-armhf-rb ;;
- ##armhf_28) NODE1=virt64a-armhf-rb NODE2=cbxi4pro0-armhf-rb ;;
- ##armhf_29) NODE1=virt32z-armhf-rb NODE2=virt64a-armhf-rb ;;
- ##armhf_30) NODE1=ff64a-armhf-rb NODE2=virt32c-armhf-rb ;;
- ##armhf_31) NODE1=virt64c-armhf-rb NODE2=ff4a-armhf-rb ;;
- ##armhf_32) NODE1=virt64c-armhf-rb NODE2=virt32c-armhf-rb ;;
- ##armhf_33) NODE1=virt32z-armhf-rb NODE2=virt64c-armhf-rb ;;
- ##armhf_34) NODE1=virt32z-armhf-rb NODE2=virt64a-armhf-rb ;;
- ##armhf_35) NODE1=cbxi4pro0-armhf-rb NODE2=virt64b-armhf-rb ;;
- *) NODE1=undefined
- ;;
- esac
+ line=$(grep "$1 " reproducible_build_workers|grep -v '^#')
+ if [ -n "$line" ]; then
+ NODE1=$(echo "$line"|awk '{print $2}')
+ NODE2=$(echo "$line"|awk '{print $3}')
+ else
+ NODE1=undefined
+ fi
}
@@ -166,6 +48,12 @@ notify_log_of_failure() {
}
main_loop() {
+ choose_nodes "$WORKER_NAME"
+ # check inside the loop, so that the worker can be disabled by just changing
+ # the file without stopping everything.
+ if [ "$NODE1" = "undefined" ]; then
+ exit 8
+ fi
LOCKFILE="/var/lib/jenkins/NO-RB-BUILDERS-PLEASE"
if [ -f "$LOCKFILE" ]; then
echo "The lockfile $LOCKFILE is present, thus stopping this"
@@ -262,13 +150,8 @@ if [ -z "${XDG_RUNTIME_DIR:-}" ]; then
XDG_RUNTIME_DIR="/run/user/$UID"
export XDG_RUNTIME_DIR
fi
-
-# script invoked without specifying the nodes
-choose_nodes "$WORKER_NAME"
SERVICE="reproducible_build@${WORKER_NAME}.service"
-if [ "$NODE1" = "undefined" ]; then
- exit 8
-fi
+
while true ; do
main_loop
done
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/fea359c6792e330c826cd4d0cc7f607b2c372e70
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/fea359c6792e330c826cd4d0cc7f607b2c372e70
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20240309/e91ca9b9/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list