[Git][qa/jenkins.debian.net][master] reproducible Debian: split scheduler job into 4 jobs for each arch
Holger Levsen (@holger)
gitlab at salsa.debian.org
Thu Jul 6 00:10:08 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
6e84a76e by Holger Levsen at 2023-07-06T01:09:36+02:00
reproducible Debian: split scheduler job into 4 jobs for each arch
(to reduce run-time of each job)
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- bin/common-functions.sh
- bin/reproducible_scheduler.py
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/common-functions.sh
=====================================
@@ -233,7 +233,7 @@ jenkins_zombie_check() {
# related however is that I issued a reboot (via running
# /sbin/reboot) right before the zombies appeared...
#
- ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | grep -E 'strip-nondeterminism|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|dpkg_buster_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|chroot-installation_buster_.*_upgrade_to_sid|piuparts_.*_(jessie|stretch|buster|bullseye)|lintian-tests_stretch|lintian-tests_buster|lintian-tests_buster-backports|lintian-tests_bullseye|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|debian-archive-keyring-tests_buster|debian-archive-keyring-tests_bullseye|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-9]($|_)|rebootstrap_.*_gcc1[01]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites|g-i-installation|reproducible_compare_Debian_sha1sums|bbx15|cb3a|ff2a|ff2b|jtk1a|jtk1b|odxu4a|odxu4b|odu3a|opi2a|opi2c|p64b|p64c|ar71xx|reproducible_debian_live_build$|chroot-installation_stretch|chroot-installation_bullseye*upgrade_to_sid|rebuilder_prototype|osuosl167|osuosl168|osuosl169|osuosl170|osuosl171|osuosl172|osuosl173|osuosl174|osuosl184fakeroot-foreign|fdroid|reproducible_.*_reproducible?$|health_check_amd64_snapshot|reproducible_.*_stretch_.*|buster_diffoscope_amd64_osuosl3|chroot-installation_buster|udd_buster_multiarch_versionskew|disorderfs_from_git|reprotest_from_git|diffoscope_from_git|reproducible_create_meta_pkg_sets$' || true)"
+ ZOMBIES="$(ls -1d /var/lib/jenkins/jobs/* | grep -E 'strip-nondeterminism|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|dpkg_buster_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|chroot-installation_buster_.*_upgrade_to_sid|piuparts_.*_(jessie|stretch|buster|bullseye)|lintian-tests_stretch|lintian-tests_buster|lintian-tests_buster-backports|lintian-tests_bullseye|udd_stretch|d-i_pu-build|debsums-tests_stretch|debian-archive-keyring-tests_stretch|debian-archive-keyring-tests_buster|debian-archive-keyring-tests_bullseye|chroot-installation_jessie|chroot-installation_.*education-lang-|kirkwoot|rebootstrap_.*_gcc[5-9]($|_)|rebootstrap_.*_gcc1[01]($|_)|brcm47xx|rebootstrap_kfreebsd|diffoscope_from_git_|disorderfs_from_git_master|diffoscope_pypi|diffoscope_freebsd|diffoscope_netbsd|diffoscope_macports|diffoscope_archlinux|openwrt-target-ath97|profitbricks|pool_buildinfos_suites|g-i-installation|reproducible_compare_Debian_sha1sums|bbx15|cb3a|ff2a|ff2b|jtk1a|jtk1b|odxu4a|odxu4b|odu3a|opi2a|opi2c|p64b|p64c|ar71xx|reproducible_debian_live_build$|chroot-installation_stretch|chroot-installation_bullseye*upgrade_to_sid|rebuilder_prototype|osuosl167|osuosl168|osuosl169|osuosl170|osuosl171|osuosl172|osuosl173|osuosl174|osuosl184fakeroot-foreign|fdroid|reproducible_.*_reproducible?$|health_check_amd64_snapshot|reproducible_.*_stretch_.*|buster_diffoscope_amd64_osuosl3|chroot-installation_buster|udd_buster_multiarch_versionskew|disorderfs_from_git|reprotest_from_git|diffoscope_from_git|reproducible_create_meta_pkg_sets$|reproducible_scheduler$' || true)"
if [ -n "$ZOMBIES" ] ; then
DIRTY=true
figlet 'zombies!!!'
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -793,6 +793,7 @@ def scheduler(arch):
return ''
if __name__ == '__main__':
+ arch = sys.argv[1]
log.info('Updating sources tables for all suites.')
for suite in SUITES:
update_sources(suite)
@@ -802,16 +803,15 @@ if __name__ == '__main__':
"FROM schedule AS p JOIN sources AS s ON s.id=p.package_id " + \
"WHERE s.architecture='{}' AND build_type='ci_build'"
message = ''
- for arch in ARCHS:
- log.info('Scheduling for %s...', arch)
- overall = int(query_db(query.format(arch))[0][0])
- if overall > (MAXIMA[arch]*3):
- log.info('%s packages already scheduled for %s, nothing to do.', overall, arch)
- continue
- log.info('%s packages already scheduled for %s, probably scheduling some '
- 'more...', overall, arch)
- message += scheduler(arch)
- log.info('Arch %s scheduled at %s.', arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+ log.info('Scheduling for %s...', arch)
+ overall = int(query_db(query.format(arch))[0][0])
+ if overall > (MAXIMA[arch]*3):
+ log.info('%s packages already scheduled for %s, nothing to do.', overall, arch)
+ continue
+ log.info('%s packages already scheduled for %s, probably scheduling some '
+ 'more...', overall, arch)
+ message += scheduler(arch)
+ log.info('Arch %s scheduled at %s.', arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
if message != '':
# build the kgb message text
message = 'Scheduled in ' + '+'.join(SUITES) + ' at ' + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ':\n' + message + '\n'
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -503,11 +503,26 @@
my_description: Build a daily backup of the database
my_timed: H H * * *
my_ shellext: .sh
- - 'scheduler':
- my_description: 'Schedule packages to be tested for reproducibility.'
+ - 'scheduler_amd64':
+ my_description: 'Schedule packages on amd64 to be tested for reproducibility.'
# dinstall runs 1|7|13|19:52 UTC - see https://salsa.debian.org/ftp-team/dak/blob/master/config/debian/crontab
- my_timed: '23 H/3 * * *'
- my_shellext: ".py"
+ my_timed: '0 H/6 * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_scheduler.py amd64'
+ - 'scheduler_arm64':
+ my_description: 'Schedule packages on arm64 to be tested for reproducibility.'
+ # dinstall runs 1|7|13|19:52 UTC - see https://salsa.debian.org/ftp-team/dak/blob/master/config/debian/crontab
+ my_timed: '15 H/6 * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_scheduler.py arm64'
+ - 'scheduler_armhf':
+ my_description: 'Schedule packages on armhf to be tested for reproducibility.'
+ # dinstall runs 1|7|13|19:52 UTC - see https://salsa.debian.org/ftp-team/dak/blob/master/config/debian/crontab
+ my_timed: '30 H/6 * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_scheduler.py armhf'
+ - 'scheduler_i386':
+ my_description: 'Schedule packages on i386 to be tested for reproducibility.'
+ # dinstall runs 1|7|13|19:52 UTC - see https://salsa.debian.org/ftp-team/dak/blob/master/config/debian/crontab
+ my_timed: '45 H/6 * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_scheduler.py i386'
- 'system_health':
my_description: 'calculate overall tests.r-b.o system health for usage with https://github.com/jelly/reproduciblebuilds-display/'
my_timed: 'H/59 * * * *'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/6e84a76e651512033fa30851e02f2298cfb033ba
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/6e84a76e651512033fa30851e02f2298cfb033ba
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230705/fe706ad8/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list