[Git][qa/jenkins.debian.net][master] 2 commits: bin/rebuilderd_stats: Break lines between sections and between packages

Holger Levsen (@holger) gitlab at salsa.debian.org
Mon Dec 16 15:55:21 GMT 2024



Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
4ec0ce17 by Gioele Barabucci at 2024-12-16T16:54:58+01:00
bin/rebuilderd_stats: Break lines between sections and between packages

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -
7ecb5025 by Gioele Barabucci at 2024-12-16T16:55:02+01:00
bin/rebuilderd_stats: Process packages only once

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- bin/rebuilderd_stats.py


Changes:

=====================================
bin/rebuilderd_stats.py
=====================================
@@ -2,7 +2,7 @@
 
 from collections import defaultdict
 from datetime import datetime
-from re import search
+from re import search, DOTALL
 from sqlite3 import connect
 from sys import argv
 
@@ -17,57 +17,72 @@ def main() -> None:
     log = "CAST(b.build_log AS TEXT)"
     r_packages = "p.name like 'r-cran-%' or p.name like 'r-bioc-%' or p.name like 'r-other-%'"
 
+    def log_has(s):
+        return lambda log, _: s in log
+
+    def log_has_re(re):
+        return lambda log, _: search(re, log, DOTALL)
+
+    def log_has_any(strs):
+        def f(log, _):
+            for s in strs:
+                if s in log:
+                    return True
+            return False
+        return f
+
+    def diff_has(s):
+        return lambda _, diff: s in (diff or '')
+
+    def diff_has_re(re):
+        return lambda _, diff: search(re, diff or '', DOTALL)
+
+    def diff_has_any(strs):
+        def f(_, diff):
+            if not diff:
+                return False
+
+            for s in strs:
+                if s in (diff or ''):
+                    return True
+            return False
+        return f
+
     error_messages = {
-        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
-        " Failed to download build input from%'": "buildinfo file 404 (maybe temporary)",
-        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
-        " Failed to download original package from%'": "package file 404 (temporary)",
-        f"{log} like '%debsnap: fatal error at line 27%'": "debsnap failed (temporary)",
-        f"{log} like '%debsnap: No source files found for%'": "debsnap failed (temporary)",
-        f"{log} like '%cannot find:%debootsnap failed"
-        " at /usr/bin/debrebuild line 48%'": "packages missing on metasnap (maybe temporary)",
-        f"{log} not like '%cannot find:%' and {log} like '%debootsnap failed"
-        " at /usr/bin/debrebuild line 48%'": "debootsnap failed (maybe temporary)",
-        f"{log} like '%Validation FAILED!!%'": "dscverify failed (temporary)",
-        f"{log} like '%400 URL must be absolute"
-        "_E: Could not download%sbuild failed%'": "download failed (temporary)",
-        f"{log} like '%E: Error creating chroot session: skipping%'": "sbuild chroot failed (temporary)",
-        f"{log} like '%E: Disk space is probably not sufficient for building.%'": "sbuild failed due to insufficient disk space",
-        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
-        f" and {log} like '%inputs/freedict_20%'": "timeout: freedict (#998683)",
-        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
-        f" and {log} not like '%inputs/freedict_20%'": "timeout (maybe temporary)",
-        f"{log} like '%TRUNCATED DUE TO SIZE LIMIT: %'": "size limit (not fatal)",
-        f"{log} like '%fakeroot not found, either install the fakeroot%'"
-        fr" and {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "old dpkg (<1.19.0, temporary)",
-        f"{log} like '%fakeroot not found, either install the fakeroot%'"
-        fr" and not {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "fakeroot not found (https://deb.li/3P46G)",
-        f"{log} not like '%fakeroot not found, either install the fakeroot%'"
-        f" and not ({r_packages})"
-        f" and {log} like '%E: Build failure (dpkg-buildpackage died)%'": "dpkg-buildpackage failed",
-        "b.diffoscope like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'": "diffoscope timeout (not fatal)",
-        f"not ({r_packages}) and (b.diffoscope is null or"
-        f" (b.diffoscope not like '%buildinfo_{arch}.gz%' and"
-        " b.diffoscope not like '%buildinfo_all.gz%' and"
-        " b.diffoscope not like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'))"
-        fr" and ({log} regexp 'checking [^ ]*: $'"
-        fr" or {log} regexp 'checking [^ ]*: size differs for [^ ]*$'"
-        fr" or {log} regexp 'checking [^ ]*: size... $'"
-        fr" or {log} regexp 'checking [^ ]*: size... value of [^ ]* differs for [^ ]*$')": "failed to reproduce",
-        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
-        " Failed to run diffoscope: No such file or directory (os error 2)%'": "diffoscope not found (fixed)",
-        r_packages: "failed to reproduce: R package (#1089197)",
-        f"b.diffoscope like '%buildinfo_{arch}.gz%' or b.diffoscope like '%buildinfo_all.gz%'": "failed to reproduce: dh_buildinfo (#1068809)",
-        f"({log} like '%.deb: size... md5... sha256... sha1... all OK_' or {log} like '%.deb: size... md5... sha1... sha256... all OK_')": "rebuilderd error"
+        "buildinfo file 404 (maybe temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download build input from"),
+        "package file 404 (temporary)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to download original package from"),
+        "debsnap failed (temporary)": log_has("debsnap: fatal error at line 27"),
+        "debsnap failed to find source files (temporary)": log_has("debsnap: No source files found for"),
+        "packages missing on metasnap (maybe temporary)": log_has_re(r"cannot find:.*debootsnap failed at /usr/bin/debrebuild line 48"),
+        "debootsnap failed (maybe temporary)": log_has("debootsnap failed at /usr/bin/debrebuild line 48"),
+        "dscverify failed (temporary)": log_has("Validation FAILED!!"),
+        "download failed (temporary)": log_has_re("400 URL must be absolute.E: Could not download.*sbuild failed"),
+        "sbuild chroot failed (temporary)": log_has("E: Error creating chroot session: skipping"),
+        "sbuild failed due to insufficient disk space": log_has("E: Disk space is probably not sufficient for building."),
+        "timeout: freedict (#998683)": log_has_any(["TRUNCATED DUE TO TIMEOUT: ", "inputs/freedict_20"]),
+        "timeout (maybe temporary)": log_has("TRUNCATED DUE TO TIMEOUT: "),
+        "size limit (not fatal)": log_has("TRUNCATED DUE TO SIZE LIMIT: "),
+        "old dpkg (<1.19.0, temporary)": log_has_re(r"dpkg is already the newest version \(1\.1[0-8].*fakeroot not found, either install the fakeroot"),
+        "fakeroot not found (https://deb.li/3P46G)": log_has("fakeroot not found, either install the fakeroot"),
+        "dpkg-buildpackage failed": log_has("E: Build failure (dpkg-buildpackage died)"),
+        "diffoscope timeout (not fatal)": diff_has("TRUNCATED DUE TO TIMEOUT: 600 seconds"),
+        "failed to reproduce: R package (#1089197)": log_has_re(r"(Source|Binary): r-(cran|bioc|other)-"),
+        "failed to reproduce: dh_buildinfo (#1068809)": diff_has_any([f"buildinfo_{arch}.gz", "buildinfo_all.gz"]),
+        "failed to reproduce": log_has_re(r"checking [^ ]*: (size(...|(value of [^ ]* differs for [^ ]*)))? ?$"),
+        "diffoscope not found (fixed)": log_has("rebuilderd: unexpected error while rebuilding package: Failed to run diffoscope: No such file or directory (os error 2)"),
+        "rebuilderd error": log_has_any([".deb: size... md5... sha256... sha1... all OK", "deb: size... md5... sha1... sha256... all OK"]),
     }
 
-    messages_packages = defaultdict(list)
-    for error, message in error_messages.items():
-        for row in cu.execute(
-            "SELECT p.name FROM packages p LEFT JOIN builds b ON b.id = p.build_id"
-            f" WHERE p.status = 'BAD' and {error}"
-        ):
-            messages_packages[message].append(row[0])
+    messages_packages = {k: [] for k in error_messages.keys()} # Preserve keys order
+    for row in cu.execute(
+        "SELECT p.name, CAST(b.build_log AS TEXT), b.diffoscope FROM packages p LEFT JOIN builds b"
+        " ON b.id = p.build_id WHERE p.status = 'BAD'"
+    ):
+        for message, error_match_fn in error_messages.items():
+            if error_match_fn(row[1], row[2]):
+                messages_packages[message].append(row[0])
+                break
+    messages_packages = {k: v for k, v in messages_packages.items() if v} # Remove empty categories
 
     package_logs = dict()
     bad_packages = set()
@@ -110,9 +125,9 @@ def main() -> None:
 
     for message, packages in messages_packages.items():
         anchor = message.replace(" ", "-")
-        print(f'<h2 id="{anchor}">{message}</h2>')
+        print(f'\n<h2 id="{anchor}">{message}</h2>')
         packages = sorted(packages, key=lambda pkg: package_logs.get(pkg, 0))
-        print(" ".join(format_link(pkg) for pkg in packages))
+        print("\n".join(format_link(pkg) for pkg in packages))
 
     print("<br/><br/><br/></main></body></html>")
 



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/b2073ef5236c8619fe3f67a7032ae10720292fc1...7ecb502521688493354d881b15d373a0051ca0f9

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/b2073ef5236c8619fe3f67a7032ae10720292fc1...7ecb502521688493354d881b15d373a0051ca0f9
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20241216/445d28e5/attachment-0001.htm>


More information about the Qa-jenkins-scm mailing list