[Git][qa/jenkins.debian.net][master] 4 commits: reproducible debian: more robust way to find the last build id

Mattia Rizzolo (@mattia) gitlab at salsa.debian.org
Sat Mar 9 15:15:51 GMT 2024



Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net


Commits:
122bf0c7 by Mattia Rizzolo at 2024-03-09T14:54:59+01:00
reproducible debian: more robust way to find the last build id

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -
16517685 by Mattia Rizzolo at 2024-03-09T14:55:57+01:00
reproducible debian: add a bunch of quotes to calm down shellcheck

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -
ad47ae3a by Mattia Rizzolo at 2024-03-09T15:05:23+01:00
reproducible debian: prefer pgrep rather than grepping ps' output

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -
dc5fd443 by Mattia Rizzolo at 2024-03-09T16:13:09+01:00
reproducible Debian: massive refactoring on how the systemd service operates

Split running the builder and running the startup thing in two pieces,
to better isolate the pieces.

Signed-off-by: Mattia Rizzolo <mattia at debian.org>

- - - - -


4 changed files:

- bin/reproducible_build_service.sh
- bin/reproducible_worker.sh
- + hosts/jenkins/etc/systemd/system/reproducible_build.service
- hosts/jenkins/etc/systemd/system/reproducible_build at .service


Changes:

=====================================
bin/reproducible_build_service.sh
=====================================
@@ -2,146 +2,15 @@
 # vim: set noexpandtab:
 
 # Copyright © 2017-2024 Holger Levsen (holger at layer-acht.org)
+# Copyright © 2018-2024 Mattia Rizzolo <mattia at debian.org>
 # released under the GPLv2
 
 set -e
 
-# global variables
-NODE1=""
-NODE2=""
-
-#
-# this function defines which builds take place on which nodes
-#
-choose_nodes() {
-	case $1 in
-		#
-		# amd64, i386 and arm64 nodes are chosen in a way that one build always runs
-		# on a node running in the future, the other on a node with correct date.
-		# armhf builds are distributed by the build capacity of the nodes, see below.
-		#
-		amd64_1)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
-		amd64_2)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
-		amd64_3)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
-		amd64_4)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
-		amd64_5)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
-		amd64_6)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
-		amd64_7)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
-		amd64_8)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
-		amd64_9)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
-		amd64_10)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
-		amd64_11)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
-		amd64_12)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
-		amd64_13)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
-		amd64_14)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
-		amd64_15)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
-		amd64_16)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
-		amd64_17)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
-		amd64_18)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
-		amd64_19)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
-		amd64_20)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
-		amd64_21)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
-		amd64_22)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
-		amd64_23)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
-		amd64_24)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
-		amd64_25)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
-		amd64_26)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
-		amd64_27)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
-		amd64_28)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
-		#amd64_29)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
-		#amd64_30)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
-		#amd64_31)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
-		#amd64_32)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
-
-		# i386
-		##i386_1)		NODE1=ionos2-i386	NODE2=ionos6-i386 ;;
-		##i386_2)		NODE1=ionos6-i386	NODE2=ionos2-i386 ;;
-		##i386_3)		NODE1=ionos2-i386	NODE2=ionos16-i386 ;;
-		##i386_4)		NODE1=ionos16-i386	NODE2=ionos2-i386 ;;
-		##i386_5)		NODE1=ionos12-i386	NODE2=ionos6-i386 ;;
-		##i386_6)		NODE1=ionos6-i386	NODE2=ionos12-i386 ;;
-		##i386_7)		NODE1=ionos12-i386	NODE2=ionos16-i386 ;;
-		##i386_8)		NODE1=ionos16-i386	NODE2=ionos12-i386 ;;
-		#i386_9)	NODE1=ionos2-i386	NODE2=ionos6-i386 ;;
-		#i386_10)	NODE1=ionos16-i386	NODE2=ionos12-i386 ;;
-		#i386_11)	NODE1=ionos6-i386	NODE2=ionos2-i386 ;;
-		#i386_12)	NODE1=ionos12-i386	NODE2=ionos16-i386 ;;
-
-		# arm64
-		arm64_1)	NODE1=codethink01-arm64	NODE2=codethink02-arm64 ;;
-		#arm64_2)	NODE1=codethink01-arm64	NODE2=codethink03-arm64 ;;
-		#arm64_3)	NODE1=codethink01-arm64	NODE2=codethink04-arm64 ;;
-		arm64_4)	NODE1=codethink02-arm64	NODE2=codethink01-arm64 ;;
-		arm64_5)	NODE1=codethink03-arm64	NODE2=codethink01-arm64 ;;
-		arm64_6)	NODE1=codethink04-arm64	NODE2=codethink01-arm64 ;;
-		arm64_7)	NODE1=codethink02-arm64	NODE2=codethink03-arm64 ;;
-		#arm64_8)	NODE1=codethink02-arm64	NODE2=codethink04-arm64 ;;
-		arm64_9)	NODE1=codethink03-arm64	NODE2=codethink02-arm64 ;;
-		arm64_10)	NODE1=codethink04-arm64	NODE2=codethink02-arm64 ;;
-		arm64_11)	NODE1=codethink03-arm64	NODE2=codethink04-arm64 ;;
-		arm64_12)	NODE1=codethink04-arm64	NODE2=codethink03-arm64 ;;
-		arm64_13)	NODE1=codethink01-arm64	NODE2=codethink02-arm64 ;;
-		arm64_14)	NODE1=codethink01-arm64	NODE2=codethink03-arm64 ;;
-		arm64_15)	NODE1=codethink01-arm64	NODE2=codethink04-arm64 ;;
-		#arm64_16)	NODE1=codethink02-arm64	NODE2=codethink01-arm64 ;;
-		#arm64_17)	NODE1=codethink03-arm64	NODE2=codethink01-arm64 ;;
-		arm64_18)	NODE1=codethink04-arm64	NODE2=codethink01-arm64 ;;
-		arm64_19)	NODE1=codethink02-arm64	NODE2=codethink03-arm64 ;;
-		arm64_20)	NODE1=codethink02-arm64	NODE2=codethink04-arm64 ;;
-		#arm64_21)	NODE1=codethink03-arm64	NODE2=codethink02-arm64 ;;
-		#arm64_22)	NODE1=codethink04-arm64	NODE2=codethink02-arm64 ;;
-		arm64_23)	NODE1=codethink03-arm64	NODE2=codethink04-arm64 ;;
-		#arm64_24)	NODE1=codethink04-arm64	NODE2=codethink03-arm64 ;;
-		# to choose new armhf jobs:
-            #   for i in cbxi4pro0 wbq0 ff64a cbxi4a cbxi4b ff4a virt32a virt32b virt32c virt32z virt64a virt64b virt64c virt64z ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf|nl ; done
-	        #       6-8 jobs for quad-cores with 15 gb ram
-	        #       6-7 jobs for quad-cores with 7 gb ram
-	        #       6 jobs for quad-cores with 4 gb ram
-	        #       4 jobs for quad-cores with 2gb of ram
-		#
-		# Don't forget to update README with the number of builders…!
-		#
-		##armhf_1)	NODE1=cbxi4a-armhf-rb		NODE2=virt64z-armhf-rb ;;
-		##armhf_2)	NODE1=virt32a-armhf-rb		NODE2=virt64c-armhf-rb ;;
-		##armhf_3)	NODE1=ff4a-armhf-rb		NODE2=virt64z-armhf-rb ;;
-		##armhf_4)	NODE1=virt64z-armhf-rb		NODE2=ff4a-armhf-rb ;;
-		##armhf_5)	NODE1=virt64a-armhf-rb		NODE2=cbxi4b-armhf-rb ;;
-		##armhf_6)	NODE1=virt64b-armhf-rb		NODE2=virt32z-armhf-rb ;;
-		##armhf_7)	NODE1=virt64b-armhf-rb		NODE2=cbxi4a-armhf-rb ;;
-		##armhf_8)	NODE1=virt64a-armhf-rb		NODE2=virt32c-armhf-rb ;;
-		##armhf_9)	NODE1=virt32a-armhf-rb		NODE2=virt64z-armhf-rb ;;
-		##armhf_10)	NODE1=wbq0-armhf-rb		NODE2=virt64a-armhf-rb ;;
-		##armhf_11)	NODE1=wbq0-armhf-rb		NODE2=virt64b-armhf-rb ;;
-		##armhf_12)	NODE1=virt64a-armhf-rb		NODE2=virt32b-armhf-rb ;;
-		##armhf_13)	NODE1=ff64a-armhf-rb		NODE2=virt32b-armhf-rb ;;
-		##armhf_14)	NODE1=virt32z-armhf-rb		NODE2=ff64a-armhf-rb ;;
-		##armhf_15)	NODE1=virt64b-armhf-rb		NODE2=virt32z-armhf-rb ;;
-		##armhf_16)	NODE1=virt32b-armhf-rb		NODE2=virt64z-armhf-rb ;;
-		##armhf_17)	NODE1=virt32b-armhf-rb		NODE2=virt64b-armhf-rb ;;
-		##armhf_18)	NODE1=ff64a-armhf-rb		NODE2=virt32a-armhf-rb ;;
-		##armhf_19)	NODE1=virt64c-armhf-rb		NODE2=cbxi4a-armhf-rb ;;
-		##armhf_20)	NODE1=virt64c-armhf-rb		NODE2=wbq0-armhf-rb ;;
-		##armhf_21)	NODE1=cbxi4a-armhf-rb		NODE2=ff64a-armhf-rb ;;
-		##armhf_22)	NODE1=cbxi4b-armhf-rb		NODE2=virt64c-armhf-rb ;;
-		##armhf_23)	NODE1=cbxi4b-armhf-rb		NODE2=ff64a-armhf-rb ;;
-		##armhf_24)	NODE1=ff4a-armhf-rb		NODE2=virt64b-armhf-rb ;;
-		##armhf_25)	NODE1=virt32c-armhf-rb		NODE2=virt64z-armhf-rb ;;
-		##armhf_26)	NODE1=virt64z-armhf-rb		NODE2=cbxi4b-armhf-rb ;;
-		##armhf_27)	NODE1=virt64z-armhf-rb		NODE2=virt32a-armhf-rb ;;
-		##armhf_28)	NODE1=virt64a-armhf-rb		NODE2=cbxi4pro0-armhf-rb ;;
-		##armhf_29)	NODE1=virt32z-armhf-rb		NODE2=virt64a-armhf-rb ;;
-		##armhf_30)	NODE1=ff64a-armhf-rb		NODE2=virt32c-armhf-rb ;;
-		##armhf_31)	NODE1=virt64c-armhf-rb		NODE2=ff4a-armhf-rb ;;
-		##armhf_32)	NODE1=virt64c-armhf-rb		NODE2=virt32c-armhf-rb ;;
-		##armhf_33)	NODE1=virt32z-armhf-rb		NODE2=virt64c-armhf-rb ;;
-		##armhf_34)	NODE1=virt32z-armhf-rb		NODE2=virt64a-armhf-rb ;;
-		##armhf_35)	NODE1=cbxi4pro0-armhf-rb	NODE2=virt64b-armhf-rb ;;
-		*)		NODE1=undefined
-				;;
-	esac
-}
-
 startup_workers() {
+	local ARCH MAX
+	local WORKER_BIN WORKER_NAME
+	local BUILD_BASE
 	#
 	# loop through all archs
 	#
@@ -156,19 +25,35 @@ startup_workers() {
 		#
 		# startup as many workers as defined above
 		#
-		for i in $(seq 1 $MAX) ; do
+		for i in $(seq 1 "$MAX") ; do
 			#
-		        # sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
+			# sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
 			#
-		        /bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
+			/bin/sleep "$(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )"
 
 			#
 			# continue loop if the worker to be started is already running
 			#
 			WORKER_NAME=${ARCH}_$i
+			#SERVICE_NAME="reproducible_build@${WORKER_NAME}.service"
 			WORKER_BIN=/srv/jenkins/bin/reproducible_worker.sh
-			RUNNING=$(ps fax|grep -v grep|grep "$WORKER_BIN $WORKER_NAME" 2>/dev/null||true)
-			if [ -n "$RUNNING" ] ; then
+
+			# first check the status of the services
+			# TODO: commented for now, let's first see how it goes without this,
+			# as it probably needs quite some tweaking
+			#local serviceSubState serviceActiveState
+			#serviceSubState=$(systemd show -P SubState "$SERVICE_NAME")
+			#serviceActiveState=$(systemd show -P ActiveState "$SERVICE_NAME")
+			#if [ "$serviceActiveState" = "active" ] && [ "$serviceSubState" = "exited" ]; then
+			#	# there is probably a good reason for this, don't restart it
+			#	echo "$(date --utc) - $WORKER_NAME already exited, not restarting.  Check log."
+			#	systemctl status "$SERVICE_NAME"
+			#	continue
+			#fi
+
+			# systemd can be in weird state, so just look at what real
+			# processes are around, don't look at what systemd think it's running
+			if pgrep -f "$WORKER_BIN $WORKER_NAME " 2> /dev/null ; then
 				echo "$(date --utc) - '$(basename $WORKER_BIN) $WORKER_NAME' already running, thus moving on to the next."
 				continue
 			fi
@@ -176,24 +61,22 @@ startup_workers() {
 			#
 			# actually start the worker
 			#
-			choose_nodes $WORKER_NAME
-			if [ "$NODE1" != "undefined" ] ; then
-				BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
-				mkdir -p $BUILD_BASE
-				echo "$(date --utc) - Starting $WORKER_NAME"
-				( setsid $WORKER_BIN $WORKER_NAME $NODE1 $NODE2 >$BUILD_BASE/worker.log 2>&1 & )
-			fi
+			BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
+			mkdir -p "$BUILD_BASE"
+			echo "$(date --utc) - Starting $WORKER_NAME"
+			systemctl start "reproducible_build@${WORKER_NAME}.service"
 		done
 	done
 }
 
 
 check_lock() {
+	local LOCKFILE children
 	LOCKFILE="/var/lib/jenkins/NO-RB-BUILDERS-PLEASE"
 	if [ -f "$LOCKFILE" ]; then
 		echo "The lockfile $LOCKFILE is present, exiting the service..."
 		while : ; do
-			children="$(pgrep --list-full --parent $$)"
+			children="$(pgrep --list-full --parent $$ || true)"
 			if [ -n "$children" ]; then
 				local SLEEPTIME=15m
 				echo "There are still some child processes, waiting $SLEEPTIME for them:"
@@ -222,5 +105,3 @@ while true ; do
 	sleep 133.7m
 	set -e
 done
-
-


=====================================
bin/reproducible_worker.sh
=====================================
@@ -2,14 +2,17 @@
 # vim: set noexpandtab:
 
 # Copyright © 2017-2018 Holger Levsen (holger at layer-acht.org)
-#           © 2018 Mattia Rizolo <mattia at debian.org>
+#           © 2018-2024 Mattia Rizolo <mattia at debian.org>
 # released under the GPLv2
 
 set -e
 
-WORKER_NAME=$1
-NODE1=$2
-NODE2=$3
+# exit status:
+# 8: worker undefined
+# 9: lockfile present
+# 10: this script is already running
+
+WORKER_NAME=${1:?worker name}
 
 # normally defined by jenkins and used by reproducible_common.sh
 JENKINS_URL=https://jenkins.debian.net
@@ -21,6 +24,138 @@ common_init "$@"
 # common code for tests.reproducible-builds.org
 . /srv/jenkins/bin/reproducible_common.sh
 
+#
+# this function defines which builds take place on which nodes
+#
+choose_nodes() {
+	case $1 in
+		#
+		# amd64, i386 and arm64 nodes are chosen in a way that one build always runs
+		# on a node running in the future, the other on a node with correct date.
+		# armhf builds are distributed by the build capacity of the nodes, see below.
+		#
+		amd64_1)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
+		amd64_2)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
+		amd64_3)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
+		amd64_4)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
+		amd64_5)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
+		amd64_6)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
+		amd64_7)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
+		amd64_8)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
+		amd64_9)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
+		amd64_10)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
+		amd64_11)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
+		amd64_12)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
+		amd64_13)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
+		amd64_14)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
+		amd64_15)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
+		amd64_16)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
+		amd64_17)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
+		amd64_18)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
+		amd64_19)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
+		amd64_20)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
+		amd64_21)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
+		amd64_22)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
+		amd64_23)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
+		amd64_24)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
+		amd64_25)	NODE1=ionos1-amd64	NODE2=ionos5-amd64 ;;
+		amd64_26)	NODE1=ionos5-amd64	NODE2=ionos1-amd64 ;;
+		amd64_27)	NODE1=ionos11-amd64	NODE2=ionos15-amd64 ;;
+		amd64_28)	NODE1=ionos15-amd64	NODE2=ionos11-amd64 ;;
+		#amd64_29)	NODE1=ionos1-amd64	NODE2=ionos15-amd64 ;;
+		#amd64_30)	NODE1=ionos15-amd64	NODE2=ionos1-amd64 ;;
+		#amd64_31)	NODE1=ionos11-amd64	NODE2=ionos5-amd64 ;;
+		#amd64_32)	NODE1=ionos5-amd64	NODE2=ionos11-amd64 ;;
+
+		# i386
+		##i386_1)		NODE1=ionos2-i386	NODE2=ionos6-i386 ;;
+		##i386_2)		NODE1=ionos6-i386	NODE2=ionos2-i386 ;;
+		##i386_3)		NODE1=ionos2-i386	NODE2=ionos16-i386 ;;
+		##i386_4)		NODE1=ionos16-i386	NODE2=ionos2-i386 ;;
+		##i386_5)		NODE1=ionos12-i386	NODE2=ionos6-i386 ;;
+		##i386_6)		NODE1=ionos6-i386	NODE2=ionos12-i386 ;;
+		##i386_7)		NODE1=ionos12-i386	NODE2=ionos16-i386 ;;
+		##i386_8)		NODE1=ionos16-i386	NODE2=ionos12-i386 ;;
+		#i386_9)	NODE1=ionos2-i386	NODE2=ionos6-i386 ;;
+		#i386_10)	NODE1=ionos16-i386	NODE2=ionos12-i386 ;;
+		#i386_11)	NODE1=ionos6-i386	NODE2=ionos2-i386 ;;
+		#i386_12)	NODE1=ionos12-i386	NODE2=ionos16-i386 ;;
+
+		# arm64
+		arm64_1)	NODE1=codethink01-arm64	NODE2=codethink02-arm64 ;;
+		#arm64_2)	NODE1=codethink01-arm64	NODE2=codethink03-arm64 ;;
+		#arm64_3)	NODE1=codethink01-arm64	NODE2=codethink04-arm64 ;;
+		arm64_4)	NODE1=codethink02-arm64	NODE2=codethink01-arm64 ;;
+		arm64_5)	NODE1=codethink03-arm64	NODE2=codethink01-arm64 ;;
+		arm64_6)	NODE1=codethink04-arm64	NODE2=codethink01-arm64 ;;
+		arm64_7)	NODE1=codethink02-arm64	NODE2=codethink03-arm64 ;;
+		#arm64_8)	NODE1=codethink02-arm64	NODE2=codethink04-arm64 ;;
+		arm64_9)	NODE1=codethink03-arm64	NODE2=codethink02-arm64 ;;
+		arm64_10)	NODE1=codethink04-arm64	NODE2=codethink02-arm64 ;;
+		arm64_11)	NODE1=codethink03-arm64	NODE2=codethink04-arm64 ;;
+		arm64_12)	NODE1=codethink04-arm64	NODE2=codethink03-arm64 ;;
+		arm64_13)	NODE1=codethink01-arm64	NODE2=codethink02-arm64 ;;
+		arm64_14)	NODE1=codethink01-arm64	NODE2=codethink03-arm64 ;;
+		arm64_15)	NODE1=codethink01-arm64	NODE2=codethink04-arm64 ;;
+		#arm64_16)	NODE1=codethink02-arm64	NODE2=codethink01-arm64 ;;
+		#arm64_17)	NODE1=codethink03-arm64	NODE2=codethink01-arm64 ;;
+		arm64_18)	NODE1=codethink04-arm64	NODE2=codethink01-arm64 ;;
+		arm64_19)	NODE1=codethink02-arm64	NODE2=codethink03-arm64 ;;
+		arm64_20)	NODE1=codethink02-arm64	NODE2=codethink04-arm64 ;;
+		#arm64_21)	NODE1=codethink03-arm64	NODE2=codethink02-arm64 ;;
+		#arm64_22)	NODE1=codethink04-arm64	NODE2=codethink02-arm64 ;;
+		arm64_23)	NODE1=codethink03-arm64	NODE2=codethink04-arm64 ;;
+		#arm64_24)	NODE1=codethink04-arm64	NODE2=codethink03-arm64 ;;
+		# to choose new armhf jobs:
+            #   for i in cbxi4pro0 wbq0 ff64a cbxi4a cbxi4b ff4a virt32a virt32b virt32c virt32z virt64a virt64b virt64c virt64z ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf|nl ; done
+	        #       6-8 jobs for quad-cores with 15 gb ram
+	        #       6-7 jobs for quad-cores with 7 gb ram
+	        #       6 jobs for quad-cores with 4 gb ram
+	        #       4 jobs for quad-cores with 2gb of ram
+		#
+		# Don't forget to update README with the number of builders…!
+		#
+		##armhf_1)	NODE1=cbxi4a-armhf-rb		NODE2=virt64z-armhf-rb ;;
+		##armhf_2)	NODE1=virt32a-armhf-rb		NODE2=virt64c-armhf-rb ;;
+		##armhf_3)	NODE1=ff4a-armhf-rb		NODE2=virt64z-armhf-rb ;;
+		##armhf_4)	NODE1=virt64z-armhf-rb		NODE2=ff4a-armhf-rb ;;
+		##armhf_5)	NODE1=virt64a-armhf-rb		NODE2=cbxi4b-armhf-rb ;;
+		##armhf_6)	NODE1=virt64b-armhf-rb		NODE2=virt32z-armhf-rb ;;
+		##armhf_7)	NODE1=virt64b-armhf-rb		NODE2=cbxi4a-armhf-rb ;;
+		##armhf_8)	NODE1=virt64a-armhf-rb		NODE2=virt32c-armhf-rb ;;
+		##armhf_9)	NODE1=virt32a-armhf-rb		NODE2=virt64z-armhf-rb ;;
+		##armhf_10)	NODE1=wbq0-armhf-rb		NODE2=virt64a-armhf-rb ;;
+		##armhf_11)	NODE1=wbq0-armhf-rb		NODE2=virt64b-armhf-rb ;;
+		##armhf_12)	NODE1=virt64a-armhf-rb		NODE2=virt32b-armhf-rb ;;
+		##armhf_13)	NODE1=ff64a-armhf-rb		NODE2=virt32b-armhf-rb ;;
+		##armhf_14)	NODE1=virt32z-armhf-rb		NODE2=ff64a-armhf-rb ;;
+		##armhf_15)	NODE1=virt64b-armhf-rb		NODE2=virt32z-armhf-rb ;;
+		##armhf_16)	NODE1=virt32b-armhf-rb		NODE2=virt64z-armhf-rb ;;
+		##armhf_17)	NODE1=virt32b-armhf-rb		NODE2=virt64b-armhf-rb ;;
+		##armhf_18)	NODE1=ff64a-armhf-rb		NODE2=virt32a-armhf-rb ;;
+		##armhf_19)	NODE1=virt64c-armhf-rb		NODE2=cbxi4a-armhf-rb ;;
+		##armhf_20)	NODE1=virt64c-armhf-rb		NODE2=wbq0-armhf-rb ;;
+		##armhf_21)	NODE1=cbxi4a-armhf-rb		NODE2=ff64a-armhf-rb ;;
+		##armhf_22)	NODE1=cbxi4b-armhf-rb		NODE2=virt64c-armhf-rb ;;
+		##armhf_23)	NODE1=cbxi4b-armhf-rb		NODE2=ff64a-armhf-rb ;;
+		##armhf_24)	NODE1=ff4a-armhf-rb		NODE2=virt64b-armhf-rb ;;
+		##armhf_25)	NODE1=virt32c-armhf-rb		NODE2=virt64z-armhf-rb ;;
+		##armhf_26)	NODE1=virt64z-armhf-rb		NODE2=cbxi4b-armhf-rb ;;
+		##armhf_27)	NODE1=virt64z-armhf-rb		NODE2=virt32a-armhf-rb ;;
+		##armhf_28)	NODE1=virt64a-armhf-rb		NODE2=cbxi4pro0-armhf-rb ;;
+		##armhf_29)	NODE1=virt32z-armhf-rb		NODE2=virt64a-armhf-rb ;;
+		##armhf_30)	NODE1=ff64a-armhf-rb		NODE2=virt32c-armhf-rb ;;
+		##armhf_31)	NODE1=virt64c-armhf-rb		NODE2=ff4a-armhf-rb ;;
+		##armhf_32)	NODE1=virt64c-armhf-rb		NODE2=virt32c-armhf-rb ;;
+		##armhf_33)	NODE1=virt32z-armhf-rb		NODE2=virt64c-armhf-rb ;;
+		##armhf_34)	NODE1=virt32z-armhf-rb		NODE2=virt64a-armhf-rb ;;
+		##armhf_35)	NODE1=cbxi4pro0-armhf-rb	NODE2=virt64b-armhf-rb ;;
+		*)		NODE1=undefined
+				;;
+	esac
+}
+
+
 notify_log_of_failure() {
 	local PKG_SUITE="$1"
 	tee -a /var/log/jenkins/reproducible-builder-errors.log <<-END
@@ -34,19 +169,18 @@ main_loop() {
 	LOCKFILE="/var/lib/jenkins/NO-RB-BUILDERS-PLEASE"
 	if [ -f "$LOCKFILE" ]; then
 		echo "The lockfile $LOCKFILE is present, thus stopping this"
-		exit
+		exit 9
 	fi
-	SERVICE="reproducible_build at startup.service"
 	# try systemctl twice, but only output and thus log the 2nd attempt…
-	RUNNING=$(systemctl show $SERVICE 2>/dev/null |grep ^SubState|cut -d "=" -f2)
+	RUNNING=$(systemctl show -P SubState "$SERVICE")
 	if [ "$RUNNING" != "running" ] ; then
 		# sometimes systemctl requests time out… handle that gracefully
 		sleep 23
-		RUNNING=$(systemctl show $SERVICE|grep ^SubState|cut -d "=" -f2)
+		RUNNING=$(systemctl show -P SubState "$SERVICE")
 		if [ "$RUNNING" != "running" ] ; then
 			echo "$(date --utc) - '$SERVICE' not running, thus stopping this."
 			sleep 42.1337m
-			exit
+			exit 1
 		fi
 	fi
 	if [ -f "$JENKINS_OFFLINE_LIST" ]; then
@@ -54,23 +188,29 @@ main_loop() {
 			if grep -q "$n" "$JENKINS_OFFLINE_LIST"; then
 				echo "$n is currently marked as offline, sleeping an hour before trying again."
 				sleep 60.1337m
-				exit
+				return
 			fi
 		done
 	fi
 
-	# sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
-	/bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
+
+	# sleep up to 2.3 seconds to help randomize startup
+	# (additionally to the random sleep reproducible_build.sh does anyway)
+	/bin/sleep "$(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )"
 
 	#
 	# increment BUILD_ID
 	#
 	BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
-	OLD_ID=$(ls -1rt $BUILD_BASE|grep -v -E "(latest|worker.log)" |sort -nr|head -1)
-	let BUILD_ID=OLD_ID+1
-	mkdir -p $BUILD_BASE/$BUILD_ID
-	rm -f $BUILD_BASE/latest
-	ln -sf $BUILD_ID $BUILD_BASE/latest
+	if [ -L "$BUILD_BASE"/latest ]; then
+		LAST_ID=$(basename "$( readlink -f latest )")
+	else
+		LAST_ID=0
+	fi
+	BUILD_ID=$(( LAST_ID + 1 ))
+	mkdir -p "$BUILD_BASE/$BUILD_ID"
+	rm -f "$BUILD_BASE/latest"
+	ln -sf "$BUILD_ID" "$BUILD_BASE/latest"
 
 	#
 	# prepare variables for export
@@ -90,27 +230,39 @@ main_loop() {
 	echo "================================================================================================"
 	echo
 	RETCODE=0
-	/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1 || RETCODE=$?
+	systemd-run --slice-inherit --send-sighup -u "rb-build-$WORKER_NAME-$BUILD_ID" \
+		/srv/jenkins/bin/reproducible_build.sh "$NODE1" "$NODE2" \
+		> "$BUILD_BASE/$BUILD_ID/console.log" 2>&1 || RETCODE=$?
 
 	if [ "$RETCODE" -ne 0 ] ; then
-		local FAILED="$(grep '^Initialising reproducibly build' $BUILD_BASE/$BUILD_ID/console.log | cut -d ' ' -f5-7)"
+		local FAILED
+		FAILED="$(grep '^Initialising reproducibly build' "$BUILD_BASE/$BUILD_ID/console.log" | cut -d ' ' -f5-7)"
 		 notify_log_of_failure "$FAILED"
 	fi
-	gzip $BUILD_BASE/$BUILD_ID/console.log || true
+	gzip "$BUILD_BASE/$BUILD_ID/console.log" || true
 	echo
 }
 
 #
 # check if we really should be running
 #
-RUNNING=$(ps fax|grep -v grep|grep "$0 $1 ")
-if [ -z "$RUNNING" ] ; then
-	echo "$(date --utc) - '$0 $1' already running, thus stopping this."
-	exit
+if pgrep -f "$0 $1 " >/dev/null ; then
+	echo "$(date --utc) - '$0 $1' already running:"
+	pgrep -f -a "$0 $1 "
+	echo "$(date --utc) - stopping the current process."
+	exit 10
 fi
+
 #
 # main
 #
+
+# script invoked without specifying the nodes
+choose_nodes "$WORKER_NAME"
+SERVICE="reproducible_build@${WORKER_NAME}.service"
+if [ "$NODE1" = "undefined" ]; then
+	exit 8
+fi
 while true ; do
 	main_loop
 done


=====================================
hosts/jenkins/etc/systemd/system/reproducible_build.service
=====================================
@@ -0,0 +1,20 @@
+[Unit]
+Description=Build service for reproducible-builds jobs
+ConditionPathExists=!/var/lib/jenkins/NO-RB-BUILDERS-PLEASE
+
+[Service]
+# Restart whenever the script exits, without rate limiting:
+Restart=always
+# special code from _build_service for when NO-RB-BUILDERS-PLEASE exists, to properly quit
+RestartPreventExitStatus=9
+StartLimitIntervalSec=0
+User=jenkins
+Group=jenkins
+ExecStart=/srv/jenkins/bin/reproducible_build_service.sh
+# this service is only starting other services, please don't kill it
+OOMPolicy=continue
+OOMScoreAdjust=-1000
+MemoryAccounting=false
+
+[Install]
+WantedBy=multi-user.target


=====================================
hosts/jenkins/etc/systemd/system/reproducible_build at .service
=====================================
@@ -5,16 +5,19 @@ ConditionPathExists=!/var/lib/jenkins/NO-RB-BUILDERS-PLEASE
 [Service]
 # Restart whenever the script exits, without rate limiting:
 Restart=always
-# special code from _build_service for when NO-RB-BUILDERS-PLEASE exists, to properly quit
-RestartPreventExitStatus=9
-StartLimitInterval=0
+# special code from _build_service for when the service should not exist, to properly quit
+RestartPreventExitStatus=8 9 10
 User=jenkins
 Group=jenkins
-ExecStart=/srv/jenkins/bin/reproducible_build_service.sh %I
+ExecStart=/srv/jenkins/bin/reproducible_worker.sh %I
+StandardOutput=append:/var/lib/jenkins/userContent/reproducible/debian/build_service/%I/worker.log
+StandardError=inherit
 SendSIGHUP=yes
-OOMPolicy=continue
-OOMScoreAdjust=-1000
-MemoryAccounting=false
+# This unit is only specific to a single builder, so it's fine to kill this.
+# Eventually the main service will restart the unit.
+# Note that diffoscope runs within this service.
+OOMPolicy=kill
+OOMScoreAdjust=100
 
 [Install]
 WantedBy=multi-user.target



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e75326235abdf9c72d240d31032699cc7c072769...dc5fd4433a9630efb340c1724f2d32a45db9b0de

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/e75326235abdf9c72d240d31032699cc7c072769...dc5fd4433a9630efb340c1724f2d32a45db9b0de
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20240309/9ff644dc/attachment-0001.htm>


More information about the Qa-jenkins-scm mailing list