[Git][qa/jenkins.debian.net][master] 3 commits: rdn: ppc64el is back, with a 2.7gb database each instead of 72
Holger Levsen (@holger)
gitlab at salsa.debian.org
Tue Jul 1 09:36:52 BST 2025
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
5d3cfe79 by Holger Levsen at 2025-07-01T10:33:55+02:00
rdn: ppc64el is back, with a 2.7gb database each instead of 72
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
76d6da2c by Jochen Sprickerhof at 2025-07-01T10:35:09+02:00
rdn stats: add simple cache
Add cache.pickle filename as the last argument.
(cherry picked from commit eaf9df6c16c0778d622decd3cf4976d1bfaa50fc)
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
ec7825b9 by Jochen Sprickerhof at 2025-07-01T10:36:38+02:00
rdn: enable stats caching
(cherry picked from commit 23f1a12e80e481ae47c0f6577044da4aceadaefb)
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
3 changed files:
- bin/rebuilderd_stats.py
- hosts/osuosl5-amd64/etc/cron.d/rebuilderd
- update_jdn.sh
Changes:
=====================================
bin/rebuilderd_stats.py
=====================================
@@ -8,7 +8,10 @@
# Licensed under GPL-2
from datetime import datetime
+from hashlib import file_digest
from os import path
+from pathlib import Path
+from pickle import dump, load
from re import DOTALL, search, sub
from sqlite3 import connect
from sys import argv
@@ -26,6 +29,20 @@ def main() -> None:
cx.create_function("regexp", 2, lambda x, y: 1 if search(x, y) else 0)
cu = cx.cursor()
+ cache_filename = Path(argv[3]) if len(argv) > 3 else Path()
+
+ hash_key = None
+ with open(argv[0], "rb") as fp:
+ hash_key = file_digest(fp, "blake2b").hexdigest()
+
+ cache = {}
+ cache_new = {}
+ if cache_filename.is_file():
+ with open(cache_filename, "rb") as fp:
+ hash_key_old, cache_old = load(fp)
+ if hash_key == hash_key_old:
+ cache = cache_old
+
def log_has(s):
return lambda log, _: s in log
@@ -101,23 +118,36 @@ def main() -> None:
unzstd = ZstdDecompressor()
- messages_packages = {k: [] for k in error_messages.keys()} # Preserve keys order
+ messages_packages: dict[str, list] = {k: [] for k in error_messages.keys()} # Preserve keys order
total = 0
for name, src_name, build_id, build_log, diffoscope in cu.execute(
"SELECT p.name, s.name, p.build_id, b.build_log, b.diffoscope"
" FROM packages p JOIN pkgbases s ON s.id = p.pkgbase_id LEFT JOIN builds b ON b.id = p.build_id"
" WHERE p.status ='BAD' ORDER BY p.build_id"
):
+ total += 1
+
+ if build_id in cache:
+ cache_new[build_id] = cache[build_id]
+ messages_packages[cache[build_id]].append({"name": name, "src": src_name, "id": build_id, "diff": diffoscope is not None})
+ continue
+
build_log = unzstd.stream_reader(build_log).read().decode("utf-8")
if diffoscope:
diffoscope = unzstd.stream_reader(diffoscope).read().decode("utf-8")
- total += 1
+
for message, error_match_fn in error_messages.items():
if error_match_fn(build_log, diffoscope):
+ cache_new[build_id] = message
messages_packages[message].append({"name": name, "src": src_name, "id": build_id, "diff": diffoscope is not None})
break
+
messages_packages = {k: v for k, v in messages_packages.items() if v} # Remove empty categories
+ if cache_filename.name:
+ with open(cache_filename, "wb") as fb:
+ dump((hash_key, cache_new), fb)
+
with urlopen("https://salsa.debian.org/reproducible-builds/reproducible-notes/-/raw/master/packages.yml?ref_type=heads") as rn:
rn_bugs = safe_load(rn)
=====================================
hosts/osuosl5-amd64/etc/cron.d/rebuilderd
=====================================
@@ -1,6 +1,4 @@
MAILTO=root
-42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db > /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done
-23 0,6,12,18 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 riscv64 ; do /srv/jenkins/bin/rebuilderd_graph.sh $arch ; done
+42 */3 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db /srv/rebuilderd/$arch/stats/cache.pickle > /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done
+23 0,6,12,18 * * * rebuilderd for arch in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do /srv/jenkins/bin/rebuilderd_graph.sh $arch ; done
-#42 */3 * * * rebuilderd for arch in all amd64 i386 arm64 armel armhf riscv64 ppc64el; do mkdir -p /srv/rebuilderd/$arch/stats/ && /srv/jenkins/bin/rebuilderd_stats.py $arch /srv/rebuilderd/$arch/rebuilderd.db > /srv/rebuilderd/$arch/stats/index.html.tmp && mv /srv/rebuilderd/$arch/stats/index.html.tmp /srv/rebuilderd/$arch/stats/index.html ; done
-#23 0,6,12,18 * * * rebuilderd for arch in all amd64 i386 arm64 armel armhf riscv64 ppc64el ; do /srv/jenkins/bin/rebuilderd_graph.sh $arch ; done
=====================================
update_jdn.sh
=====================================
@@ -748,7 +748,7 @@ if [ "$HOSTNAME" = "jenkins" ] || [ "$HOSTNAME" = "ionos7-amd64" ] || [ "$HOSTNA
fi
# set secret for remote rebuilderd-workers
case $HOSTNAME in
- osuosl5*) for a in all amd64 arm64 armel armhf i386 riscv64 ppc64el ; do
+ osuosl5*) for a in all amd64 arm64 armel armhf i386 ppc64el riscv64 ; do
SIGNUPSECRET="$(sudo cat /opt/etc/rebuilderd-${a}.secret)"
f=rebuilderd-${a}.conf
if [ -f /opt/etc/$f ] ; then
@@ -1011,12 +1011,9 @@ case $HOSTNAME in
*ppc64el) deploy_rebuilderd_services worker 2
;;
osuosl5*)
- deploy_rebuilderd_services rebuilderd "all amd64 arm64 armel armhf i386 riscv64"
- deploy_rebuilderd_services pkgsync "all amd64 arm64 armel armhf i386 riscv64"
- deploy_rebuilderd_services db-backup "all amd64 arm64 armel armhf i386 riscv64"
- #deploy_rebuilderd_services rebuilderd "all amd64 arm64 armel armhf i386 riscv64 ppc64el"
- #deploy_rebuilderd_services pkgsync "all amd64 arm64 armel armhf i386 riscv64 ppc64el"
- #deploy_rebuilderd_services db-backup "all amd64 arm64 armel armhf i386 riscv64 ppc64el"
+ deploy_rebuilderd_services rebuilderd "all amd64 arm64 armel armhf i386 ppc64el riscv64"
+ deploy_rebuilderd_services pkgsync "all amd64 arm64 armel armhf i386 ppc64el riscv64"
+ deploy_rebuilderd_services db-backup "all amd64 arm64 armel armhf i386 ppc64el riscv64"
sudo systemctl enable --now rebuilderd-db-backup-compress.timer
;;
*) ;;
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9fd922a3a3baa382777e1cf0b10aca0b84fad8e2...ec7825b9819c4d839c9a69ca549cd9c439bdb8f1
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/9fd922a3a3baa382777e1cf0b10aca0b84fad8e2...ec7825b9819c4d839c9a69ca549cd9c439bdb8f1
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20250701/5382a3a1/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list