[Git][qa/jenkins.debian.net][master] 2 commits: reproducible debian: scheduler: squash some stylistic issues flagged by flake8
Mattia Rizzolo (@mattia)
gitlab at salsa.debian.org
Thu Mar 14 11:51:27 GMT 2024
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
094f7cda by Mattia Rizzolo at 2024-03-13T19:35:53+01:00
reproducible debian: scheduler: squash some stylistic issues flagged by flake8
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
c9027bad by Mattia Rizzolo at 2024-03-14T12:50:23+01:00
reproducible debian: scheduler: stop scheduling unstable and exp on armhf until the t64 transition improves
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
1 changed file:
- bin/reproducible_scheduler.py
Changes:
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -2,8 +2,8 @@
# -*- coding: utf-8 -*-
#
# Copyright © 2015-2023 Mattia Rizzolo <mattia at mapreri.org>
-# Copyright © 2015-2022 Holger Levsen <holger at layer-acht.org>
-# Based on reproducible_scheduler.sh © 2014-2015 Holger Levsen <holger at layer-acht.org>
+# Copyright © 2014-2022 Holger Levsen <holger at layer-acht.org>
+#
# Licensed under GPL-2
#
# Depends: python3 python3-debian
@@ -20,13 +20,12 @@ from debian.debian_support import Version
from rblib import query_db, db_table
from rblib.confparse import log
-from rblib.const import SUITES, ARCHS, conn_db
+from rblib.const import SUITES, conn_db
from rblib.utils import print_critical_message
from rblib.models import Package
from reproducible_html_live_status import generate_schedule
from reproducible_html_packages import gen_packages_html
-from reproducible_html_packages import purge_old_pages, purge_old_suite_arch_pages
-
+from reproducible_html_packages import purge_old_suite_arch_pages
"""
@@ -87,19 +86,19 @@ LIMITS = {
'unstable': {'*': 255},
'experimental': {'*': 255},
},
- 'arm64': {
+ 'arm64': {
'bullseye': {'*': 255},
'bookworm': {'*': 255},
'trixie': {'*': 255},
'unstable': {'*': 255},
'experimental': {'*': 255},
},
- 'armhf': {
+ 'armhf': {
'bullseye': {'*': 255},
'bookworm': {'*': 255},
'trixie': {'*': 255},
- 'unstable': {'*': 255},
- 'experimental': {'*': 255},
+ 'unstable': {'*': 0},
+ 'experimental': {'*': 0},
},
},
'new': {
@@ -128,8 +127,8 @@ LIMITS = {
'bullseye': {1: (900, 300), 2: (1400, 450), '*': 75},
'bookworm': {1: (900, 400), 2: (1400, 600), '*': 75},
'trixie': {1: (900, 400), 2: (1400, 600), '*': 75},
- 'unstable': {1: (900, 450), 2: (1400, 650), '*': 150},
- 'experimental': {1: (900, 400), 2: (1400, 600), '*': 25},
+ 'unstable': {1: (900, 0), 2: (1400, 0), '*': 0},
+ 'experimental': {1: (900, 0), 2: (1400, 0), '*': 0},
},
},
'ftbfs': {
@@ -158,8 +157,8 @@ LIMITS = {
'bullseye': {1: (900, 0), 2: (1400, 0), '*': 5},
'bookworm': {1: (900, 0), 2: (1400, 0), '*': 5},
'trixie': {1: (900, 20), 2: (1400, 10), '*': 5},
- 'unstable': {1: (900, 20), 2: (1400, 10), '*': 5},
- 'experimental': {1: (900, 20), 2: (1400, 10), '*': 2},
+ 'unstable': {1: (900, 0), 2: (1400, 0), '*': 0},
+ 'experimental': {1: (900, 0), 2: (1400, 0), '*': 0},
}
},
'depwait': {
@@ -188,8 +187,8 @@ LIMITS = {
'bullseye': {1: (900, 100), 2: (1400, 50), '*': 50},
'bookworm': {1: (900, 100), 2: (1400, 50), '*': 50},
'trixe': {1: (900, 200), 2: (1400, 100), '*': 50},
- 'unstable': {1: (900, 200), 2: (1400, 100), '*': 50},
- 'experimental': {1: (900, 200), 2: (1400, 100), '*': 20},
+ 'unstable': {1: (900, 0), 2: (1400, 0), '*': 0},
+ 'experimental': {1: (900, 0), 2: (1400, 0), '*': 0},
}
},
'old': {
@@ -218,8 +217,8 @@ LIMITS = {
'bullseye': {1: (900, 0), 2: (1400, 0), '*': 0},
'bookworm': {1: (1000, 0), 2: (1500, 0), '*': 0},
'trixie': {1: (1000, 900), 2: (1500, 600), '*': 0},
- 'unstable': {1: (900, 1200), 2: (1400, 800), '*': 0},
- 'experimental': {1: (900, 70), 2: (1400, 50), '*': 0},
+ 'unstable': {1: (900, 0), 2: (1400, 0), '*': 0},
+ 'experimental': {1: (900, 0), 2: (1400, 0), '*': 0},
}
}
}
@@ -282,7 +281,9 @@ def update_sources(suite, arch):
log.debug('\tdownloaded')
log.info('Updating sources db for %s/%s...', suite, arch)
update_sources_db(suite, arch, sources)
- log.info('DB update done for %s/%s done at %s.', suite, arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
+ log.info(
+ 'DB update done for %s/%s done at %s.',
+ suite, arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def update_sources_db(suite, arch, sources):
@@ -296,7 +297,7 @@ def update_sources_db(suite, arch, sources):
log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg))
continue
- # only keep the most recent version of a src for each package/suite/arch
+ # only keep the most recent version of a src for each suite/arch
key = src['Package'] + suite + arch
if key in newest_version:
oldversion = newest_version[key]
@@ -309,7 +310,9 @@ def update_sources_db(suite, arch, sources):
# get the current packages in the database
query = "SELECT name, version, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
- cur_pkgs = set([(p.name, p.version, p.suite, p.architecture) for p in query_db(query)])
+ cur_pkgs = set(
+ [(p.name, p.version, p.suite, p.architecture) for p in query_db(query)]
+ )
pkgs_to_add = []
updated_pkgs = []
different_pkgs = [x for x in new_pkgs if x not in cur_pkgs]
@@ -438,15 +441,23 @@ def queue_packages(all_pkgs, packages, date):
date = date.strftime('%Y-%m-%d %H:%M')
pkgs = [x for x in packages if x[0] not in all_pkgs]
if len(pkgs) > 0:
- log.info('The following ' + str(len(pkgs)) + ' source packages have ' +
- 'been queued up for scheduling at ' + date + ': ' +
- ' '.join([str(x[1]) for x in pkgs]))
+ log.info(
+ 'The following ' + str(len(pkgs)) + ' source packages have ' +
+ 'been queued up for scheduling at ' + date + ': ' +
+ ' '.join([str(x[1]) for x in pkgs])
+ )
all_pkgs.update({x[0]: date for x in pkgs})
return all_pkgs
def schedule_packages(packages):
- pkgs = [{'package_id': x, 'date_scheduled': packages[x], 'build_type': 'ci_build'} for x in packages.keys()]
+ pkgs = [
+ {
+ 'package_id': x,
+ 'date_scheduled': packages[x],
+ 'build_type': 'ci_build'
+ } for x in packages.keys()
+ ]
log.debug('IDs about to be scheduled: %s', packages.keys())
if pkgs:
conn_db.execute(db_table('schedule').insert(), pkgs)
@@ -464,11 +475,20 @@ def query_untested_packages(suite, arch, limit):
query = """SELECT DISTINCT *
FROM (
SELECT sources.id, sources.name FROM sources
- WHERE sources.suite='{suite}' AND sources.architecture='{arch}'
+ WHERE sources.suite='{suite}'
+ AND sources.architecture='{arch}'
AND NOT exists
- (SELECT schedule.package_id FROM schedule WHERE build_type='ci_build' AND sources.id = schedule.package_id )
+ (
+ SELECT schedule.package_id
+ FROM schedule
+ WHERE build_type='ci_build'
+ AND sources.id = schedule.package_id
+ )
AND NOT exists
- (SELECT FROM results WHERE sources.id = results.package_id )
+ (
+ SELECT FROM results
+ WHERE sources.id = results.package_id
+ )
ORDER BY random()
) AS tmp
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
@@ -478,22 +498,28 @@ def query_untested_packages(suite, arch, limit):
def query_new_versions(suite, arch, limit):
- criteria = 'tested before, new version available, sorted by last build date'
- query = """SELECT s.id, s.name, s.version, r.version, max(r.build_date) max_date
- FROM sources AS s JOIN results AS r ON s.id = r.package_id
- WHERE s.suite='{suite}' AND s.architecture='{arch}'
- AND s.version != r.version
- AND r.status != 'blacklisted'
- AND s.id IN (SELECT package_id FROM results)
- AND NOT exists (SELECT schedule.package_id FROM schedule WHERE build_type='ci_build' AND s.id = schedule.package_id)
- GROUP BY s.id, s.name, s.version, r.version
- ORDER BY max_date
- LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
+ criteria = \
+ 'tested before, new version available, sorted by last build date'
+ query = """
+ SELECT s.id, s.name, s.version, r.version, max(r.build_date) max_date
+ FROM sources AS s JOIN results AS r ON s.id = r.package_id
+ WHERE s.suite='{suite}' AND s.architecture='{arch}'
+ AND s.version != r.version
+ AND r.status != 'blacklisted'
+ AND s.id IN (SELECT package_id FROM results)
+ AND NOT exists (
+ SELECT schedule.package_id
+ FROM schedule
+ WHERE build_type='ci_build' AND s.id = schedule.package_id
+ )
+ GROUP BY s.id, s.name, s.version, r.version
+ ORDER BY max_date
+ LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
pkgs = query_db(query)
# the next line avoids constant rescheduling of packages:
# packages in our repository != official repo,
- # so they will always be selected by the query above
- # so we only accept them if there version is greater than the already tested one
+ # so they will always be selected by the query above, to prevent this
+ # we accept them if their version is greater than the already tested one
packages = [(x[0], x[1]) for x in pkgs if Version(x[2]) > Version(x[3])]
print_schedule_result(suite, arch, criteria, packages)
return packages
@@ -510,7 +536,10 @@ def query_old_ftbfs_versions(suite, arch, limit):
AND r.status='FTBFS'
AND ( n.bugs = '[]' OR n.bugs IS NULL )
AND r.build_date < '{date}'
- AND NOT exists (SELECT FROM schedule WHERE build_type='ci_build' AND s.id = schedule.package_id )
+ AND NOT exists (
+ SELECT FROM schedule
+ WHERE build_type='ci_build' AND s.id = schedule.package_id
+ )
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -529,7 +558,10 @@ def query_old_depwait_versions(suite, arch, limit):
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status='depwait'
AND r.build_date < '{date}'
- AND NOT exists (SELECT FROM schedule WHERE build_type='ci_build' AND s.id = schedule.package_id )
+ AND NOT exists (
+ SELECT FROM schedule
+ WHERE build_type='ci_build' AND s.id = schedule.package_id
+ )
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -540,16 +572,20 @@ def query_old_depwait_versions(suite, arch, limit):
def query_old_versions(suite, arch, limit):
- criteria = """tested at least {minimum_age} days ago, no new version available,
- sorted by last build date""".format(minimum_age=MINIMUM_AGE[arch])
- date = (datetime.now()-timedelta(days=MINIMUM_AGE[arch]))\
- .strftime('%Y-%m-%d %H:%M')
+ criteria = f"tested at least {MINIMUM_AGE[arch]} days ago, no new " +\
+ "version available, sorted by last build date"
+ date = (
+ datetime.now()-timedelta(days=MINIMUM_AGE[arch])
+ ).strftime('%Y-%m-%d %H:%M')
query = """SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status != 'blacklisted'
AND r.build_date < '{date}'
- AND NOT exists (SELECT FROM schedule WHERE build_type='ci_build' AND s.id = schedule.package_id )
+ AND NOT exists (
+ SELECT FROM schedule
+ WHERE build_type='ci_build' AND s.id = schedule.package_id
+ )
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch,
@@ -558,6 +594,7 @@ def query_old_versions(suite, arch, limit):
print_schedule_result(suite, arch, criteria, packages)
return packages
+
def query_e404_versions(suite, arch, limit):
criteria = """tested at least 12h ago, status E404,
sorted by last build date"""
@@ -567,7 +604,10 @@ def query_e404_versions(suite, arch, limit):
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status = 'E404'
AND r.build_date < '{date}'
- AND NOT exists (SELECT FROM schedule WHERE build_type='ci_build' AND s.id = schedule.package_id )
+ AND NOT exists (
+ SELECT FROM schedule
+ WHERE build_type='ci_build' AND s.id = schedule.package_id
+ )
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -576,6 +616,7 @@ def query_e404_versions(suite, arch, limit):
print_schedule_result(suite, arch, criteria, packages)
return packages
+
def schedule_untested_packages(arch, total):
packages = {}
limit = Limit(arch, 'untested')
@@ -585,9 +626,11 @@ def schedule_untested_packages(arch, total):
log.info('Requesting %s untested packages in %s/%s...',
many_untested, suite, arch)
packages[suite] = query_untested_packages(suite, arch, many_untested)
- log.info('Received ' + str(len(packages[suite])) +
- ' untested packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info(
+ f"Received %s untested packages in {suite}/{arch} to schedule.",
+ len(packages[suite])
+ )
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' new packages'
@@ -607,7 +650,7 @@ def schedule_new_versions(arch, total):
packages[suite] = query_new_versions(suite, arch, many_new)
log.info('Received ' + str(len(packages[suite])) +
' new packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' new versions'
@@ -625,9 +668,11 @@ def schedule_old_ftbfs_versions(arch, total):
log.info('Requesting %s old ftbfs packages in %s/%s...', old_ftbfs,
suite, arch)
packages[suite] = query_old_ftbfs_versions(suite, arch, old_ftbfs)
- log.info('Received ' + str(len(packages[suite])) +
- ' old ftbfs packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info(
+ f"Received %s old FTBFS packages in {suite}/{arch} to schedule.",
+ len(packages[suite])
+ )
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' ftbfs without bugs filed'
@@ -645,9 +690,11 @@ def schedule_old_depwait_versions(arch, total):
log.info('Requesting %s old depwait packages in %s/%s...', old_depwait,
suite, arch)
packages[suite] = query_old_depwait_versions(suite, arch, old_depwait)
- log.info('Received ' + str(len(packages[suite])) +
- ' old depwait packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info(
+ f"Received %s old depwait packages in {suite}/{arch} to schedule.",
+ len(packages[suite])
+ )
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' in depwait state'
@@ -667,7 +714,7 @@ def schedule_old_versions(arch, total):
packages[suite] = query_old_versions(suite, arch, many_old)
log.info('Received ' + str(len(packages[suite])) +
' old packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' known versions'
@@ -675,6 +722,7 @@ def schedule_old_versions(arch, total):
msg = ''
return packages, msg
+
def schedule_e404_versions(arch, total):
packages = {}
for suite in SUITES:
@@ -683,7 +731,7 @@ def schedule_e404_versions(arch, total):
packages[suite] = query_e404_versions(suite, arch, LIMIT_E404)
log.info('Received ' + str(len(packages[suite])) +
' E404 packages in ' + suite + '/' + arch + ' to schedule.')
- log.info('--------------------------------------------------------------')
+ log.info('-----------------------------------------------------------')
msg = add_up_numbers(packages, arch)
if msg != '0':
msg += ' with status \'E404\''
@@ -698,7 +746,7 @@ def scheduler(arch):
"WHERE s.architecture='{arch}' AND build_type='ci_build'"
total = int(query_db(query.format(arch=arch))[0][0])
log.info('==============================================================')
- log.info('Currently scheduled packages in all suites on ' + arch + ': ' + str(total))
+ log.info(f"Currently scheduled packages in all suites on {arch}: {total}")
if total > MAXIMA[arch]:
log.info(str(total) + ' packages already scheduled' +
', only scheduling new versions.')
@@ -731,7 +779,8 @@ def scheduler(arch):
for suite in priotized_suite_order:
query = "SELECT count(*) " \
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " \
- "WHERE s.suite='{suite}' AND s.architecture='{arch}' AND p.build_type='ci_build'"
+ "WHERE s.suite='{suite}' " \
+ "AND s.architecture='{arch}' AND p.build_type='ci_build'"
query = query.format(suite=suite, arch=arch)
now_queued_here[suite] = int(query_db(query)[0][0]) + \
len(untested[suite]+new[suite]+old[suite])
@@ -772,6 +821,7 @@ def scheduler(arch):
return message + '\n'
return ''
+
if __name__ == '__main__':
arch = sys.argv[1]
log.info('Updating sources tables for all suites.')
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/554d7eb2bfdd4d5b4dc5a3ca044007c44491cf91...c9027bad6af5f51d3dd2e2f4c2df891952e4702e
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/compare/554d7eb2bfdd4d5b4dc5a3ca044007c44491cf91...c9027bad6af5f51d3dd2e2f4c2df891952e4702e
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20240314/b925ec7e/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list