[Qa-jenkins-scm] [jenkins.debian.net] 02/05: reproducible/openwrt|lede: use multinode setup for lede and openwrt
Holger Levsen
holger at layer-acht.org
Sat Jul 30 19:47:13 UTC 2016
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 4f5e06648460ad557f00033b4cc8a2c1b1794cc7
Author: Alexander Couzens <lynxis at fe80.eu>
Date: Sat Jul 30 20:59:52 2016 +0200
reproducible/openwrt|lede: use multinode setup for lede and openwrt
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
bin/reproducible_lede.sh | 12 +---
bin/reproducible_openwrt.sh | 20 +-----
bin/reproducible_openwrt_common.sh | 123 +++++++++++++++++++++++++------------
3 files changed, 87 insertions(+), 68 deletions(-)
diff --git a/bin/reproducible_lede.sh b/bin/reproducible_lede.sh
index 05aaa6d..534056c 100755
--- a/bin/reproducible_lede.sh
+++ b/bin/reproducible_lede.sh
@@ -42,20 +42,10 @@ START=$(date +'%s')
trap cleanup_tmpdirs INT TERM EXIT
cd $TMPBUILDDIR
-echo "============================================================================="
-echo "$(date -u) - Cloning LEDE git repository."
-echo "============================================================================="
-git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO lede
-cd lede
-OPENWRT="$(git log -1)"
-OPENWRT_VERSION=$(git describe --always)
-echo "This is lede $OPENWRT_VERSION."
-echo
-git log -1
create_results_dirs lede
-build_two_times lede ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n"
+build_two_times lede ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" profitbricks-build3-amd64 profitbricks-build4-amd64
# for now we only build one architecture until it's at most reproducible
#build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n"
diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh
index a5fab82..d343590 100755
--- a/bin/reproducible_openwrt.sh
+++ b/bin/reproducible_openwrt.sh
@@ -42,27 +42,11 @@ START=$(date +'%s')
trap cleanup_tmpdirs INT TERM EXIT
cd $TMPBUILDDIR
-echo "============================================================================="
-echo "$(date -u) - Cloning OpenWrt git repository."
-echo "============================================================================="
-git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO openwrt
-cd openwrt
-OPENWRT="$(git log -1)"
-OPENWRT_VERSION=$(git describe --always)
-echo "This is openwrt $OPENWRT_VERSION."
-echo
-git log -1
-
-# External feeds disabled for now as they break building (especially with CONFIG_ALL=y).
-#echo "============================================================================="
-#echo "$(date -u) - Updating package feeds."
-#echo "============================================================================="
-#./scripts/feeds update -a
-#./scripts/feeds install -a
create_results_dirs openwrt
-build_two_times openwrt ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n"
+build_two_times openwrt ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" profitbricks-build3-amd64 profitbricks-build4-amd64
+
# for now we only build one architecture until it's at most reproducible
#build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n"
diff --git a/bin/reproducible_openwrt_common.sh b/bin/reproducible_openwrt_common.sh
index 1fc8430..b3f4e31 100644
--- a/bin/reproducible_openwrt_common.sh
+++ b/bin/reproducible_openwrt_common.sh
@@ -5,6 +5,15 @@
# 2016 Alexander Couzens <lynxis at fe80.eu>
# released under the GPLv=2
+# only called direct on a remote build node
+openwrt_cleanup_tmpdirs() {
+ export TMPDIR=$1
+ export TMPBUILDDIR=$TMPDIR/build
+ cleanup_tmpdirs
+}
+
+# called as trap handler
+# called on cleanup
cleanup_tmpdirs() {
cd
# (very simple) check we are deleting the right stuff
@@ -23,13 +32,11 @@ create_results_dirs() {
mkdir -p $BASE/$project/dbd
}
-# project = openwrt or lede
-# postfix = we use the postfix to save difference of the first and second build
save_logs() {
- local project="$1"
- local postfix="$2"
+ local TYPE=$1
+ local RUN=$2
- tar cJf "$BASE/${project}/dbd/logs_${postfix}.tar.xz" logs/
+ tar cJf "$TMPDIR/$RUN/logs_${TYPE}.tar.xz" logs/
}
# RUN - is b1 or b2. b1 for first run, b2 for second
@@ -173,6 +180,13 @@ openwrt_compile() {
ionice -c 3 $MAKE $OPTIONS package/index || true # don't let some packages fail the whole build
}
+openwrt_get_banner() {
+ TMPDIR=$1
+ TYPE=$2
+ cd $TMPDIR/build/$TYPE
+ cat $(find build_dir/ -name banner | grep etc/banner|head -1| xargs cat /dev/null)
+}
+
openwrt_cleanup() {
rm build_dir/target-* -rf
rm staging_dir/target-* -rf
@@ -180,54 +194,85 @@ openwrt_cleanup() {
rm logs/* -rf
}
-# TARGET a target including subtarget. E.g. ar71xx_generic
-# CONFIG - a simple basic .config as string. Use \n to seperate lines
+# openwrt_build is run on a remote host
# TYPE - openwrt or lede
+# RUN - b1 or b2. b1 means first run, b2 second
+# TARGET - a target including subtarget. E.g. ar71xx_generic
+# CONFIG - a simple basic .config as string. Use \n to seperate lines
+# TMPPATH - is a unique path generated with mktmp
# lede has a different output directory than openwrt
-build_two_times() {
- TYPE=$1
- TARGET=$2
- CONFIG=$3
+openwrt_build() {
+ local TYPE=$1
+ local RUN=$2
+ local TARGET=$3
+ local CONFIG=$4
+ export TMPDIR=$5
+ export TMPBUILDDIR=$TMPDIR/build/
+ mkdir -p $TMPBUILDDIR
- openwrt_config $CONFIG
- openwrt_build_toolchain
+ # we have also to set the TMP
- # FIRST BUILD
- openwrt_apply_variations b1
- openwrt_compile "$TYPE" b1 "$TARGET"
+ cd $TMPBUILDDIR
- # get banner
- cat $(find build_dir/ -name banner | grep etc/banner|head -1) > $BANNER_HTML
+ # checkout the repo
+ echo "============================================================================="
+ echo "$(date -u) - Cloning $TYPE git repository."
+ echo "============================================================================="
+ git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO $TYPE
+ cd $TYPE
- # save results in b1
- [ "$TYPE" = "lede" ] && save_lede_results b1
- [ "$TYPE" = "openwrt" ] && save_openwrt_results b1
+ # set tz, date, core, ..
+ openwrt_apply_variations $RUN
+
+ # configure openwrt
+ openwrt_config $CONFIG
+ openwrt_build_toolchain
+ # build images and packages
+ openwrt_compile $TYPE $RUN $TARGET
+
+ # save the results
+ [ "$TYPE" = "lede" ] && save_lede_results $RUN
+ [ "$TYPE" = "openwrt" ] && save_openwrt_results $RUN
# copy logs
- save_logs $TYPE b1
+ save_logs $TYPE $RUN
# clean up between builds
openwrt_cleanup
+}
- # SECOND BUILD
- openwrt_apply_variations b2
- openwrt_compile "$TYPE" b2 "$TARGET"
+# build openwrt/lede on two different hosts
+# TARGET a target including subtarget. E.g. ar71xx_generic
+# CONFIG - a simple basic .config as string. Use \n to seperate lines
+# TYPE - openwrt or lede
+# lede has a different output directory than openwrt
+build_two_times() {
+ TYPE=$1
+ TARGET=$2
+ CONFIG=$3
+ HOST_B1=$4
+ HOST_B2=$5
- # save results in b2
- [ "$TYPE" = "lede" ] && save_lede_results b2
- [ "$TYPE" = "openwrt" ] && save_openwrt_results b2
+ ## HOST_B1
+ RUN=b1
+ TMPDIR_B1=$(ssh $HOST_B1 mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-lede-build-XXXXXXXX)
+ # TODO check tmpdir exist
- # copy logs
- save_logs $TYPE b2
+ SCRIPT="$0 slave"
+ ssh $HOST_B1 $SCRIPT openwrt_build $TYPE $RUN $TARGET $CONFIG $TMPDIR_B1
- # reset environment to default values again
- export LANG="en_GB.UTF-8"
- unset LC_ALL
- export TZ="/usr/share/zoneinfo/UTC"
- export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:"
- umask 0022
+ # rsync back
+ # copy logs and images
+ rsync -a $HOST_B1:$TMPDIR_B1/$RUN/ $TMPDIR/$RUN/
- # clean up again
- openwrt_cleanup
-}
+ ssh $HOST_B1 $SCRIPT openwrt_get_banner $TMPDIR_B1 $TYPE > $BANNER_HTML
+ ssh $HOST_B1 $SCRIPT openwrt_cleanup_tmpdirs $TMPDIR_B1
+ ## HOST_B2
+ RUN=b2
+ TMPDIR_B2=$(ssh $HOST_A mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-lede-build-XXXXXXXX)
+ ssh $HOST_B2 $SCRIPT openwrt_build $TYPE $RUN $TARGET $CONFIG $TMPDIR_B2
+
+ rsync -a $HOST_B2:$TMPDIR_B2/$RUN/ $TMPDIR/$RUN/
+ ssh $HOST_B2 $SCRIPT openwrt_cleanup_tmpdirs $TMPDIR_B2
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list