[Git][qa/jenkins.debian.net][master] reproduce.debian.net: add rebuilder_stats.py

Holger Levsen (@holger) gitlab at salsa.debian.org
Mon Dec 9 19:23:41 GMT 2024



Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net


Commits:
a4bf4abb by Jochen Sprickerhof at 2024-12-09T20:22:11+01:00
reproduce.debian.net: add rebuilder_stats.py

taken from https://salsa.debian.org/-/snippets/753

Signed-off-by: Holger Levsen <holger at layer-acht.org>

- - - - -


1 changed file:

- + bin/rebuilder_stats.py


Changes:

=====================================
bin/rebuilder_stats.py
=====================================
@@ -0,0 +1,119 @@
+#!/usr/bin/python3
+
+from collections import defaultdict
+from datetime import datetime
+from re import search
+from sqlite3 import connect
+from sys import argv
+
+
+def main() -> None:
+    arch = argv[1]
+    cx = connect("/var/lib/jenkins/rebuilderd.db")
+    cx.create_function("regexp", 2, lambda x, y: 1 if search(x, y) else 0)
+    cu = cx.cursor()
+
+    log = "CAST(b.build_log AS TEXT)"
+    r_packages = "p.name like 'r-cran-%' or p.name like 'r-bioc-%' or p.name like 'r-other-%'"
+
+    error_messages = {
+        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
+        " Failed to download build input from%'": "buildinfo file 404 (maybe temporary)",
+        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
+        " Failed to download original package from%'": "package file 404 (temporary)",
+        f"{log} like '%debsnap: fatal error at line 27%'": "debsnap failed (temporary)",
+        f"{log} like '%debsnap: No source files found for%'": "debsnap failed (temporary)",
+        f"{log} like '%cannot find:%debootsnap failed"
+        " at /usr/bin/debrebuild line 48%'": "packages missing on metasnap (maybe temporary)",
+        f"{log} not like '%cannot find:%' and {log} like '%debootsnap failed"
+        " at /usr/bin/debrebuild line 48%'": "debootsnap failed (maybe temporary)",
+        f"{log} like '%Validation FAILED!!%'": "dscverify failed (temporary)",
+        f"{log} like '%400 URL must be absolute"
+        "_E: Could not download%sbuild failed%'": "download failed (temporary)",
+        f"{log} like '%E: Error creating chroot session: skipping%'": "sbuild chroot failed (temporary)",
+        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
+        f" and {log} like '%inputs/freedict_20%'": "timeout: freedict #998683",
+        f"{log} like '%TRUNCATED DUE TO TIMEOUT: %'"
+        f" and {log} not like '%inputs/freedict_20%'": "timeout",
+        f"{log} like '%TRUNCATED DUE TO SIZE LIMIT: %'": "size limit",
+        f"{log} like '%fakeroot not found, either install the fakeroot%'"
+        fr" and {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "old dpkg",
+        f"{log} like '%fakeroot not found, either install the fakeroot%'"
+        fr" and not {log} regexp 'dpkg is already the newest version \(1\.1[0-8]'": "fakeroot not found",
+        f"{log} not like '%fakeroot not found, either install the fakeroot%'"
+        f" and not ({r_packages})"
+        f" and {log} like '%E: Build failure (dpkg-buildpackage died)%'": "dpkg-buildpackage failed",
+        "b.diffoscope like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'": "diffoscope timeout",
+        f"not ({r_packages}) and (b.diffoscope is null or"
+        f" (b.diffoscope not like '%buildinfo_{arch}.gz%' and"
+        " b.diffoscope not like '%buildinfo_all.gz%' and"
+        " b.diffoscope not like '%TRUNCATED DUE TO TIMEOUT: 600 seconds%'))"
+        fr" and ({log} regexp 'checking [^ ]*: $'"
+        fr" or {log} regexp 'checking [^ ]*: size differs for [^ ]*$'"
+        fr" or {log} regexp 'checking [^ ]*: size... $'"
+        fr" or {log} regexp 'checking [^ ]*: size... value of [^ ]* differs for [^ ]*$')": "failed to reproduce",
+        f"{log} like '%rebuilderd: unexpected error while rebuilding package:"
+        " Failed to run diffoscope: No such file or directory (os error 2)%'": "diffoscope not found (fixed)",
+        r_packages: "failed to reproduce: R package #1089197",
+        f"b.diffoscope like '%buildinfo_{arch}.gz%' or b.diffoscope like '%buildinfo_all.gz%'": "dh_buildinfo",
+        f"({log} like '%.deb: size... md5... sha256... sha1... all OK_' or {log} like '%.deb: size... md5... sha1... sha256... all OK_')": "rebuilderd error"
+    }
+
+    messages_packages = defaultdict(list)
+    for error, message in error_messages.items():
+        for row in cu.execute(
+            "SELECT p.name FROM packages p LEFT JOIN builds b ON b.id = p.build_id"
+            f" WHERE p.status = 'BAD' and {error}"
+        ):
+            messages_packages[message].append(row[0])
+
+    package_logs = dict()
+    bad_packages = set()
+    has_diffoscope = set()
+    for row in cu.execute("SELECT p.name, p.build_id, p.has_diffoscope FROM packages p WHERE p.status ='BAD'"):
+        bad_packages.add(row[0])
+        package_logs[row[0]] = int(row[1])
+        if row[2]:
+            has_diffoscope.add(row[0])
+    found_packages = {pkg for lst in messages_packages.values() for pkg in lst}
+
+    other_errors = list(bad_packages.difference(found_packages))
+    if other_errors:
+        messages_packages["other errors"] = other_errors
+
+    print(
+        '<!DOCTYPE html><html lang="en"><head>'
+        '<meta charset="utf-8">'
+        f"<title>https://{arch}.reproduce.debian.net/ stats</title>"
+        '<meta name="viewport" content="width=device-width, initial-scale=1">'
+        "</head><body>"
+        f"<header><h1>https://{arch}.reproduce.debian.net/ stats</h1></header> <main>"
+    )
+    print(f"Last changed: {datetime.now().isoformat()}")
+
+    print("<table> <tr> <th>error</th> <th>number of affected bad packages</th> </tr>")
+    for message, packages in messages_packages.items():
+        anchor = message.replace(" ", "-")
+        print(
+            f'<tr><td><a href="#{anchor}">{message}</a></td>'
+            f'<td>{len(packages)} ({len(packages)/len(bad_packages)*100:.2f}%)</td></tr>'
+        )
+    print("</table>")
+
+    def format_link(pkg):
+        link = f'<a href="https://{arch}.reproduce.debian.net/api/v0/builds/{package_logs.get(pkg, 0)}/log">{pkg}</a><a href="https://tracker.debian.org/pkg/{pkg}">🍥</a>'
+        if pkg in has_diffoscope:
+            return f'{link}<a href="https://{arch}.reproduce.debian.net/api/v0/builds/{package_logs.get(pkg, 0)}/diffoscope">💠</a>'
+        return link
+
+    for message, packages in messages_packages.items():
+        anchor = message.replace(" ", "-")
+        print(f'<h2 id="{anchor}">{message}</h2>')
+        packages = sorted(packages, key=lambda pkg: package_logs.get(pkg, 0))
+        print(" ".join(format_link(pkg) for pkg in packages))
+
+    print("</main></body></html>")
+
+
+if __name__ == "__main__":
+    main()



View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/a4bf4abbf01a042009eea8f3dadefe1957a5de75

-- 
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/a4bf4abbf01a042009eea8f3dadefe1957a5de75
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20241209/5c75f83f/attachment-0001.htm>


More information about the Qa-jenkins-scm mailing list