[Reproducible-builds] Patch V2 for build nodes pools

Sat Dec 19 02:34:17 UTC 2015

On 2015-12-18, Holger Levsen wrote:
> Hi Vagrant,
>
> On Samstag, 19. Dezember 2015, Vagrant Cascadian wrote:
>> Patch below! No idea if it works, given that I don't have a spare
>> jenkins.debian.net or build network to test on, but hopefully it
>> demonstrates the idea, and is mostly there.
>
> as discussed on irc: thanks for your work on this! I like the idea, I very 
> much like the idea of developing this as a 16th (or N+1th) build job and in 
> general I'm willing to give it a try.

I didn't spend any time really figuring out which nodes to add to the
example 16th build job, so that might need some adjusting.


> Just please address those "FIXME"s first, and second the concerns I've
> raised so far on IRC, and third, future concerns ;-)

Ok, updated patch should address those concerns and a few obvious bugfixes:

- Split load estimating into it's own script, and add support for
available memory.

- Only set NODE* and PORT* from build_rebuild when called from
build_rebuild function.

- .debian.net on NODE1 and NODE2 in build_rebuild function.

- Call timeout so that the ssh processes don't take too long to complete.

diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 338c207..28dd8ed 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -688,9 +688,43 @@ check_buildinfo() {
 	rm -f $TMPFILE1 $TMPFILE2
 }
 
+select_least_loaded_node() {
+    local pool_nodes
+    local node
+    local port
+    local load
+    local best_load
+    local selected
+    # default to the first node
+    selected="$1"
+    pool_nodes="$@"
+    if [ "$selected" = "$pool_nodes" ]; then
+	echo $selected
+	return 0
+    fi
+    load = 0
+    best_load = 0
+    for this_node in $pool_nodes ; do
+	node=$(echo $this_node | cut -d : -f 1)
+	port=$(echo $this_node | cut -d : -f 2)
+	load=$(timeout 10 ssh $node -p $port /srv/jenkins/bin/reproducible_build_node_load.sh)
+	if [ "$load" -gt "$best_load" ]; then
+	    selected="$this_node"
+	    best_load="$load"
+	fi
+    done
+    echo $selected
+}
+
 build_rebuild() {
 	FTBFS=1
 	mkdir b1 b2
+	local selected_node
+	selected_node=$(select_least_loaded_node $NODE1_POOL)
+	NODE1=$(echo $selected_node | cut -d : -f 1).debian.net
+	PORT1=$(echo $selected_node | cut -d : -f 2)
+	# if no port is given, assume 22
+	if [ "$NODE1" = "${PORT1}.debian.net" ] ; then PORT1=22 ; fi
 	remote_build 1 $NODE1 $PORT1
 	if [ ! -f b1/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] && [ -f b1/${SRCPACKAGE}_*_${ARCH}.changes ] ; then
 			echo "Version mismatch between main node (${SRCPACKAGE}_${EVERSION}_${ARCH}.dsc expected) and first build node ($(ls b1/*dsc)) for $SUITE/$ARCH, aborting. Please upgrade the schroots..." | tee -a ${RBUILDLOG}
@@ -700,6 +734,11 @@ build_rebuild() {
 			exit 0
 	elif [ -f b1/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then
 		# the first build did not FTBFS, try rebuild it.
+		selected_node=$(select_least_loaded_node $NODE2_POOL)
+		NODE2=$(echo $selected_node | cut -d : -f 1).debian.net
+		PORT2=$(echo $selected_node | cut -d : -f 2)
+		# if no port is given, assume 22
+		if [ "$NODE2" = "${PORT2}.debian.net" ] ; then PORT2=22 ; fi
 		remote_build 2 $NODE2 $PORT2
 		if [ -f b2/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then
 			# both builds were fine, i.e., they did not FTBFS.
@@ -750,17 +789,12 @@ elif [ "$1" = "1" ] || [ "$1" = "2" ] ; then
 	exit 0
 elif [ "$2" != "" ] ; then
 	MODE="master"
-	NODE1="$(echo $1 | cut -d ':' -f1).debian.net"
-	NODE2="$(echo $2 | cut -d ':' -f1).debian.net"
-	PORT1="$(echo $1 | cut -d ':' -f2)"
-	PORT2="$(echo $2 | cut -d ':' -f2)"
-	# if no port is given, assume 22
-	if [ "$NODE1" = "${PORT1}.debian.net" ] ; then PORT1=22 ; fi
-	if [ "$NODE2" = "${PORT2}.debian.net" ] ; then PORT2=22 ; fi
+	NODE1_POOL="$1"
+	NODE2_POOL="$2"
 	# overwrite ARCH for remote builds
 	for i in $ARCHS ; do
 		# try to match ARCH in nodenames
-		if [[ "$NODE1" =~ .*-$i.* ]] ; then
+		if [[ "$NODE1_POOL" =~ .*-$i.* ]] ; then
 			ARCH=$i
 		fi
 	done
diff --git a/bin/reproducible_node_load.sh b/bin/reproducible_node_load.sh
new file mode 100755
index 0000000..36bce84
--- /dev/null
+++ b/bin/reproducible_node_load.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# produce an estimated load based on the number of CPUs and RAM, minus
+# the load average.
+
+cpus=$(grep ^processor /proc/cpuinfo | wc -l)
+memory_available=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
+loadavg=$(awk '{print $1}' /proc/loadavg)
+fudge=100
+
+# Calculation is basically, memory + cpus - loadavg, with some fudge
+# factor to try to ensure it's only a positive number.
+echo "$memory_available + ($cpus - $loadavg + $fudge) * 1000000" | \
+    bc | \
+    cut -d . -f 1
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 97df606..5963206 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -481,7 +481,8 @@
                 - '13': { my_node1: 'cbxi4pro0-armhf-rb:2226', my_node2: 'rpi2b-armhf-rb:2230'     }
                 - '14': { my_node1: 'rpi2b-armhf-rb:2230',     my_node2: 'wbq0-armhf-rb:2225'      }
                 - '15': { my_node1: 'wbd0-armhf-rb:2223',      my_node2: 'hb0-armhf-rb:2224'       }
-            my_shell: '/srv/jenkins/bin/reproducible_build.sh {my_node1} {my_node2}'
+                - '16': { my_node1: 'wbd0-armhf-rb:2223 wbq0-armhf-r:2225', my_node2: 'bpi0-armhf-rb:2222 odxu4-armhf-rb:2229' }
+            my_shell: '/srv/jenkins/bin/reproducible_build.sh "{my_node1}" "{my_node2}"'
             my_timed: '* * * * *'
             my_hname: ''
 

> btw, there's jenkins-test-vm.debian.net now too, but currently it's mostly 
> only suited for testing job configurations…

Ah well, someday... :)


live well,
  vagrant
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 818 bytes
Desc: not available
URL: <http://lists.alioth.debian.org/pipermail/reproducible-builds/attachments/20151218/f92fb786/attachment.sig>
