[Qa-jenkins-scm] [jenkins.debian.net] 01/01: reproducible Debian: refactor build service so that it only needs once service instead of 166
Holger Levsen
holger at layer-acht.org
Wed Apr 26 10:29:21 UTC 2017
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 54cfe059808eb26b802a3f6d15ee2150083300fe
Author: Holger Levsen <holger at layer-acht.org>
Date: Wed Apr 26 12:28:54 2017 +0200
reproducible Debian: refactor build service so that it only needs once service instead of 166
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
bin/reproducible_build_service.sh | 102 +++++++++++++++----------------
bin/reproducible_build_service_worker.sh | 44 +++++++++++++
2 files changed, 93 insertions(+), 53 deletions(-)
diff --git a/bin/reproducible_build_service.sh b/bin/reproducible_build_service.sh
index 0d3e260..4f80158 100755
--- a/bin/reproducible_build_service.sh
+++ b/bin/reproducible_build_service.sh
@@ -3,65 +3,62 @@
# Copyright © 2017 Holger Levsen (holger at layer-acht.org)
# released under the GPLv=2
-# normally defined by jenkins
-JENKINS_URL=https://jenkins.debian.net
set -e
set -x
-# sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
-/bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
-DEBUG=false
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
-# common code defining db access
-. /srv/jenkins/bin/reproducible_common.sh
+choose_node() {
+ case $1 in
+ arm64_1) NODE1=codethink-sled12-arm64 NODE2=codethink-sled15-arm64 ;;
+ i386_1) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_2) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_3) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_4) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_5) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_6) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
+ i386_7) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_8) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
+ i386_9) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_10) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_11) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_12) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_13) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_14) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
+ i386_15) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_16) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
+ i386_17) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_18) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_19) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_20) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
+ i386_21) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
+ i386_22) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
+ i386_23) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
+ i386_24) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
+ *) echo "Sleeping 60min"
+ sleep 60m
+ exit 0
+ ;;
+ esac
+}
-BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$1
-OLD_ID=$(ls -1rt $BUILD_BASE|grep -v latest|sort -n|tail -1)
-let BUILD_ID=OLD_ID+1
-mkdir -p $BUILD_BASE/$BUILD_ID
-rm -f $BUILD_BASE/latest
-ln -sf $BUILD_ID $BUILD_BASE/latest
+NODE1=""
+NODE2=""
+ARCH=i386
+# let's start with two workers only…
+for i in $(seq 1 2) ; do
+ # sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
+ /bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
-export BUILD_URL=https://jenkins.debian.net/userContent/build_service/$1/
-export BUILD_ID=$BUILD_ID
-export JOB_NAME="reproducible_builder_$1"
-export
+ WORKER_NAME=${ARCH}_$i
+ choose_node $WORKER_NAME
+ BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
+ mkdir -p $BUILD_BASE
+ echo "$(date --utc) - Starting $WORKER_NAME"
+ /srv/jenkins/bin/reproducible_build_service_worker.sh $WORKER_NAME $NODE1 $NODE2 >$BUILD_BASE/worker.log 2>&1 &
+done
-case $1 in
- arm64_1) NODE1=codethink-sled12-arm64 NODE2=codethink-sled15-arm64 ;;
- i386_1) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
- i386_2) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
- i386_3) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
- i386_4) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
- i386_5) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
- i386_6) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
- i386_7) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
- i386_8) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
- i386_9) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
- i386_10) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
- i386_11) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
- i386_12) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
- i386_13) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
- i386_14) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
- i386_15) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
- i386_16) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
- i386_17) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build6-i386 ;;
- i386_18) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build2-i386 ;;
- i386_19) NODE1=profitbricks-build2-i386 NODE2=profitbricks-build16-i386 ;;
- i386_20) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build2-i386 ;;
- i386_21) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build6-i386 ;;
- i386_22) NODE1=profitbricks-build6-i386 NODE2=profitbricks-build12-i386 ;;
- i386_23) NODE1=profitbricks-build12-i386 NODE2=profitbricks-build16-i386 ;;
- i386_24) NODE1=profitbricks-build16-i386 NODE2=profitbricks-build12-i386 ;;
- *) echo "Sleeping 60min"
- sleep 60m
- exit 0
- ;;
-esac
-
-/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1
+# not sure if this is really needed…
+while true ; do sleep 5m ; done
# TODO left:
# * maintenance job needs to:
@@ -70,5 +67,4 @@ esac
# * logs should auto display in browser like with jenkins… (long-polling, meta-refresh, something)
# - there's an NPH solution pointed out by Xtaran
# * translate yaml into a script or such to create those service files (done for i386 for now)
-# * have one service per arch, starting X build jobs instead?
diff --git a/bin/reproducible_build_service_worker.sh b/bin/reproducible_build_service_worker.sh
new file mode 100755
index 0000000..b057b77
--- /dev/null
+++ b/bin/reproducible_build_service_worker.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright © 2017 Holger Levsen (holger at layer-acht.org)
+# released under the GPLv=2
+
+# normally defined by jenkins
+JENKINS_URL=https://jenkins.debian.net
+set -e
+set -x
+
+WORKER_NAME=$1
+NODE1=$2
+NODE2=$3
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+while true ; do
+ # TODO
+ # - test here if the builder service is actually running…
+
+ # sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
+ /bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
+
+ BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
+ OLD_ID=$(ls -1rt $BUILD_BASE|egrep -v "(latest|worker.log)" |sort -n|tail -1)
+ let BUILD_ID=OLD_ID+1
+ mkdir -p $BUILD_BASE/$BUILD_ID
+ rm -f $BUILD_BASE/latest
+ ln -sf $BUILD_ID $BUILD_BASE/latest
+
+ export BUILD_URL=https://jenkins.debian.net/userContent/build_service/$WORKER_NAME/
+ export BUILD_ID=$BUILD_ID
+ export JOB_NAME="reproducible_builder_$WORKER_NAME"
+ export
+
+ /srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1
+done
+
+
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list