[Git][qa/jenkins.debian.net][master] 4 commits: reproducible Debian: speed up html indexes, thanks to Myon! <3
Holger Levsen (@holger)
gitlab at salsa.debian.org
Mon Jul 10 21:04:58 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
3c55d007 by Holger Levsen at 2023-07-10T17:24:15+02:00
reproducible Debian: speed up html indexes, thanks to Myon! <3
explaination in https://wiki.postgresql.org/wiki/Don%27t_Do_This#Don.27t_use_NOT_IN
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
19114739 by Holger Levsen at 2023-07-10T22:01:57+02:00
fixup typo in reproducible Debian: speed up scheduler a lot, thanks to Myon! <3
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
db5b3f1a by Holger Levsen at 2023-07-10T22:03:20+02:00
reproducible Debian: speed up dashboard job, thanks to Myon! <3
explaination in https://wiki.postgresql.org/wiki/Don%27t_Do_This#Don.27t_use_NOT_IN
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
7009e616 by Holger Levsen at 2023-07-10T22:04:37+02:00
reproducible Debian: speed up query to get usertagged bugs, thanks to Myon! <3
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
5 changed files:
- bin/rblib/bugs.py
- bin/reproducible_common.sh
- bin/reproducible_html_dashboard.sh
- bin/reproducible_html_indexes.py
- bin/reproducible_scheduler.py
Changes:
=====================================
bin/rblib/bugs.py
=====================================
@@ -84,10 +84,12 @@ class Bugs:
) AS tags ON bugs.id = tags.id
WHERE
bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org'
- AND bugs.id NOT IN (
- SELECT id
+ AND NOT exists (
+ SELECT
FROM bugs_usertags
- WHERE email = 'reproducible-builds at lists.alioth.debian.org'
+ WHERE
+ id = bugs.id
+ AND email = 'reproducible-builds at lists.alioth.debian.org'
AND (
bugs_usertags.tag = 'toolchain'
OR bugs_usertags.tag = 'infrastructure')
=====================================
bin/reproducible_common.sh
=====================================
@@ -351,7 +351,7 @@ write_page_header() {
write_page " <a href=\"$JENKINS_URL/userContent/about.html#_reproducible_builds_jobs\">jenkins.debian.net</a>."
write_page " Thanks to <a href=\"https://www.ionos.com\">IONOS</a> for donating the virtual machines this is running on!"
write_page "</ul>"
- LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status = 'FTBR' AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
+ LATEST=$(query_db "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status = 'FTBR' AND s.suite = 'unstable' AND s.architecture = 'amd64' AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1)
write_page "<form action=\"$REPRODUCIBLE_URL/redirect\" method=\"GET\">$REPRODUCIBLE_URL/"
write_page "<input type=\"text\" name=\"SrcPkg\" placeholder=\"Type my friend..\" value=\"$LATEST\" />"
write_page "<input type=\"submit\" value=\"submit source package name\" />"
=====================================
bin/reproducible_html_dashboard.sh
=====================================
@@ -621,20 +621,20 @@ create_dashboard_page() {
gather_suite_arch_stats
TD_PKG_SID_ISSUES="$TD_PKG_SID_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_NOISSUES="$TD_PKG_SID_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(bc_round "($RESULT*100/$COUNT_TOTAL)")%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_FTBR="$TD_PKG_SID_FTBR<td>$RESULT / $(bc_round "($RESULT*100/$COUNT_TOTAL)")%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_SID_FTBFS="$TD_PKG_SID_FTBFS<td>$RESULT / $(bc_round "($RESULT*100/$COUNT_TOTAL)")%</td>"
SUITE="${TESTING_SUITE}"
gather_suite_arch_stats
TD_PKG_TESTING_ISSUES="$TD_PKG_TESTING_ISSUES<td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_TESTING_NOISSUES="$TD_PKG_TESTING_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBR' AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_TESTING_FTBR="$TD_PKG_TESTING_FTBR<td>$RESULT / $(bc_round "($RESULT*100/$COUNT_TOTAL)")%</td>"
- RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
+ RESULT=$(query_db "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='$SUITE' AND s.architecture='$ARCH') tmp")
TD_PKG_TESTING_FTBFS="$TD_PKG_TESTING_FTBFS<td>$RESULT / $(bc_round "($RESULT*100/$COUNT_TOTAL)")%</td>"
done
write_page "$TD_PKG_SID_NOISSUES</tr>"
=====================================
bin/reproducible_html_indexes.py
=====================================
@@ -623,7 +623,7 @@ pages = {
'notes_hint': True,
'title': 'Packages without notes',
'header': '<p>There are {tot} faulty packages without notes in {suite}/{arch}.{hint}</p>',
- 'header_query': "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE s.distribution={distro} AND r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='{suite}' AND s.architecture='{arch}') AS tmp",
+ 'header_query': "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE s.distribution={distro} AND r.status IN ('FTBR', 'FTBFS', 'blacklisted') AND NOT exists (SELECT FROM notes WHERE package_id = s.id ) AND s.suite='{suite}' AND s.architecture='{arch}') AS tmp",
'body': [
{
'status': Status.FTBR,
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -488,7 +488,7 @@ def query_untested_packages(suite, arch, limit):
AND NOT exists
(SELECT schedule.package_id FROM schedule WHERE build_type='ci_build' AND sources.id = schedule.package_id )
AND NOT exists
- (SELECT FROM results AND sources.id = results.package_id )
+ (SELECT FROM results WHERE sources.id = results.package_id )
ORDER BY random()
) AS tmp
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/c27c9ddf30da95813775d69db20fa99b833f54a7...7009e6160787e5625d31f89cd6fec1092a1bd9b1
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/c27c9ddf30da95813775d69db20fa99b833f54a7...7009e6160787e5625d31f89cd6fec1092a1bd9b1
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230710/41cba16e/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list