[Git][qa/jenkins.debian.net][master] 2 commits: bin/rebuilderd_stats: Break lines between sections and between packages
Holger Levsen (@holger)
gitlab at salsa.debian.org
Mon Dec 16 15:55:21 GMT 2024
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
4ec0ce17 by Gioele Barabucci at 2024-12-16T16:54:58+01:00
bin/rebuilderd_stats: Break lines between sections and between packages
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7ecb5025 by Gioele Barabucci at 2024-12-16T16:55:02+01:00
bin/rebuilderd_stats: Process packages only once
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
1 changed file:
- bin/rebuilderd_stats.py
Changes:
=====================================
bin/rebuilderd_stats.py
=====================================
@@ -2,7 +2,7 @@
from collections import defaultdict
from datetime import datetime
-from re import search
+from re import search, DOTALL
from sqlite3 import connect
from sys import argv
@@ -17,57 +17,72 @@ def main() -> None:
log = "CAST(b.build_log AS TEXT)"
r_packages = "p.name like 'r-cran-%' or p.name like 'r-bioc-%' or p.name like 'r-other-%'"
+ def log_has(s):
+ return lambda log, _: s in log
+
+ def log_has_re(re):
+ return lambda log, _: search(re, log, DOTALL)
+
+ def log_has_any(strs):
+ def f(log, _):
+ for s in strs:
+ if s in log:
+ return True
+ return False
+ return f
+
+ def diff_has(s):
+ return lambda _, diff: s in (diff or '')
+
+ def diff_has_re(re):
+ return lambda _, diff: search(re, diff or '', DOTALL)
+
+ def diff_has_any(strs):
+ def f(_, diff):
+ if not diff:
+ return False
+
+ for s in strs:
+ if s in (diff or ''):
+ return True
+ return False
+ return f
+
error_messages = {
- f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
- " Failed to download build input from%'": "buildinfo file 404 (maybe temporary)",
- f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
- " Failed to download original package from%'": "package file 404 (temporary)",
- f"{log} like '%debsnap: fatal error at line 27%'": "debsnap failed (temporary)",
- f"{log} like '%debsnap: No source files found for%'": "debsnap failed (temporary)",
- f"{log} like '%cannot find:%debootsnap failed"
- " at /usr/bin/debrebuild line 48%'": "packages missing on metasnap (maybe temporary)",
- f"{log} not like '%cannot find:%' and {log} like '%debootsnap failed"
- " at /usr/bin/debrebuild line 48%'": "debootsnap failed (maybe temporary)",
- f"{log} like '%Validation FAILED!!%'": "dscverify failed (temporary)",
- f"{log} like '%400 URL must be absolute"
- "_E: Could not download%sbuild failed%'": "download failed (temporary)",
- f"{log} like '%E: Error creating chroot session: skipping%'": "sbuild chroot failed (temporary)",
- f"{log} like '%E: Disk space is probably not sufficient for building.%'": "sbuild failed due to insufficient disk space",
- f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
- f" and {log} like '%inputs/freedict_20%'": "timeout: freedict (#998683)",
- f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
- f" and {log} not like '%inputs/freedict_20%'": "timeout (maybe temporary)",
- f"{log} like '%TRUNCATED DUE TO SIZE LIMIT: %'": "size limit (not fatal)",
- f"{log} like '%fakeroot not found, either install the fakeroot%'"
- fr" and {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "old dpkg (<1.19.0, temporary)",
- f"{log} like '%fakeroot not found, either install the fakeroot%'"
- fr" and not {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "fakeroot not found (https://deb.li/3P46G)",
- f"{log} not like '%fakeroot not found, either install the fakeroot%'"
- f" and not ({r_packages})"
- f" and {log} like '%E: Build failure (dpkg-buildpackage died)%'": "dpkg-buildpackage failed",
- "b.diffoscope like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'": "diffoscope timeout (not fatal)",
- f"not ({r_packages}) and (b.diffoscope is null or"
- f" (b.diffoscope not like '%buildinfo_{arch}.gz%' and"
- " b.diffoscope not like '%buildinfo_all.gz%' and"
- " b.diffoscope not like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'))"
- fr" and ({log} regexp 'checking [^ ]*: $'"
- fr" or {log} regexp 'checking [^ ]*: size differs for [^ ]*$'"
- fr" or {log} regexp 'checking [^ ]*: size... $'"
- fr" or {log} regexp 'checking [^ ]*: size... value of [^ ]* differs for [^ ]*$')": "failed to reproduce",
- f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
- " Failed to run diffoscope: No such file or directory (os error 2)%'": "diffoscope not found (fixed)",
- r_packages: "failed to reproduce: R package (#1089197)",
- f"b.diffoscope like '%buildinfo_{arch}.gz%' or b.diffoscope like '%buildinfo_all.gz%'": "failed to reproduce: dh_buildinfo (#1068809)",
- f"({log} like '%.deb: size... md5... sha256... sha1... all OK_' or {log} like '%.deb: size... md5... sha1... sha256... all OK_')": "rebuilderd error"
+ "buildinfo file 404 (maybe temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download build input from"),
+ "package file 404 (temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download original package from"),
+ "debsnap failed (temporary)": log_has("debsnap: fatal error at line 27"),
+ "debsnap failed to find source files (temporary)": log_has("debsnap: No source files found for"),
+ "packages missing on metasnap (maybe temporary)": log_has_re(r"cannot find:.*debootsnap failed at /usr/bin/debrebuild line 48"),
+ "debootsnap failed (maybe temporary)": log_has("debootsnap failed at /usr/bin/debrebuild line 48"),
+ "dscverify failed (temporary)": log_has("Validation FAILED!!"),
+ "download failed (temporary)": log_has_re("400 URL must be absolute.E: Could not download.*sbuild failed"),
+ "sbuild chroot failed (temporary)": log_has("E: Error creating chroot session: skipping"),
+ "sbuild failed due to insufficient disk space": log_has("E: Disk space is probably not sufficient for building."),
+ "timeout: freedict (#998683)": log_has_any(["TRUNCATED DUE TO TIMEOUT: ", "inputs/freedict_20"]),
+ "timeout (maybe temporary)": log_has("TRUNCATED DUE TO TIMEOUT: "),
+ "size limit (not fatal)": log_has("TRUNCATED DUE TO SIZE LIMIT: "),
+ "old dpkg (<1.19.0, temporary)": log_has_re(r"dpkg is already the newest version \(1\.1[0-8].*fakeroot not found, either install the fakeroot"),
+ "fakeroot not found (https://deb.li/3P46G)": log_has("fakeroot not found, either install the fakeroot"),
+ "dpkg-buildpackage failed": log_has("E: Build failure (dpkg-buildpackage died)"),
+ "diffoscope timeout (not fatal)": diff_has("TRUNCATED DUE TO TIMEOUT: 600 seconds"),
+ "failed to reproduce: R package (#1089197)": log_has_re(r"(Source|Binary): r-(cran|bioc|other)-"),
+ "failed to reproduce: dh_buildinfo (#1068809)": diff_has_any([f"buildinfo_{arch}.gz", "buildinfo_all.gz"]),
+ "failed to reproduce": log_has_re(r"checking [^ ]*: (size(...|(value of [^ ]* differs for [^ ]*)))? ?$"),
+ "diffoscope not found (fixed)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to run diffoscope: No such file or directory (os error 2)"),
+ "rebuilderd error": log_has_any([".deb: size... md5... sha256... sha1... all OK", "deb: size... md5... sha1... sha256... all OK"]),
}
- messages_packages = defaultdict(list)
- for error, message in error_messages.items():
- for row in cu.execute(
- "SELECT p.name FROM packages p LEFT JOIN builds b ON b.id = p.build_id"
- f" WHERE p.status = 'BAD' and {error}"
- ):
- messages_packages[message].append(row[0])
+ messages_packages = {k: [] for k in error_messages.keys()} # Preserve keys order
+ for row in cu.execute(
+ "SELECT p.name, CAST(b.build_log AS TEXT), b.diffoscope FROM packages p LEFT JOIN builds b"
+ " ON b.id = p.build_id WHERE p.status = 'BAD'"
+ ):
+ for message, error_match_fn in error_messages.items():
+ if error_match_fn(row[1], row[2]):
+ messages_packages[message].append(row[0])
+ break
+ messages_packages = {k: v for k, v in messages_packages.items() if v} # Remove empty categories
package_logs = dict()
bad_packages = set()
@@ -110,9 +125,9 @@ def main() -> None:
for message, packages in messages_packages.items():
anchor = message.replace(" ", "-")
- print(f'<h2 id="{anchor}">{message}</h2>')
+ print(f'\n<h2 id="{anchor}">{message}</h2>')
packages = sorted(packages, key=lambda pkg: package_logs.get(pkg, 0))
- print(" ".join(format_link(pkg) for pkg in packages))
+ print("\n".join(format_link(pkg) for pkg in packages))
print("<br/><br/><br/></main></body></html>")
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/b2073ef5236c8619fe3f67a7032ae10720292fc1...7ecb502521688493354d881b15d373a0051ca0f9
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/b2073ef5236c8619fe3f67a7032ae10720292fc1...7ecb502521688493354d881b15d373a0051ca0f9
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20241216/445d28e5/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list