[Git][qa/jenkins.debian.net][master] reproducible Debian scheduler: refactoring
Holger Levsen (@holger)
gitlab at salsa.debian.org
Thu Jul 6 00:36:32 BST 2023
Holger Levsen pushed to branch master at Debian QA / jenkins.debian.net
Commits:
f7bd6aa0 by Holger Levsen at 2023-07-06T01:36:19+02:00
reproducible Debian scheduler: refactoring
Signed-off-by: Holger Levsen <holger at layer-acht.org>
- - - - -
2 changed files:
- bin/reproducible_html_packages.py
- bin/reproducible_scheduler.py
Changes:
=====================================
bin/reproducible_html_packages.py
=====================================
@@ -461,63 +461,66 @@ def gen_all_rb_pkg_pages(no_clean=False):
gen_packages_html(pkgs, no_clean=True) # we clean at the end
purge_old_pages()
+def purge_old_suite_arch_pages(suite, arch):
+ log.info('Removing old pages from ' + suite + '/' + arch + '.')
+ try:
+ presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
+ arch))
+ except OSError as e:
+ if e.errno != errno.ENOENT: # that's 'No such file or
+ raise # directory' error (errno 17)
+ presents = []
+ log.debug('page presents: ' + str(presents))
+
+ # get the existing packages
+ query = (
+ "SELECT s.name, s.suite, s.architecture "
+ "FROM sources s JOIN distributions d on d.id=s.distribution "
+ "WHERE s.suite=:suite AND s.architecture=:arch "
+ "AND d.name=:dist"
+ )
+ cur_pkgs = set([
+ (p.name, p.suite, p.architecture) for p in query_db(
+ sqlalchemy.text(query), suite=suite, arch=arch, dist=DISTRO)
+ ])
+
+ for page in presents:
+ # When diffoscope results exist for a package, we create a page
+ # that displays the diffoscope results by default in the main iframe
+ # in this subdirectory. Ignore this directory.
+ if page == 'diffoscope-results':
+ continue
+ pkg = page.rsplit('.', 1)[0]
+
+ if (pkg, suite, arch) not in cur_pkgs:
+ log.info('There is no package named ' + pkg + ' from ' +
+ suite + '/' + arch + ' in the database. ' +
+ 'Removing old page.')
+ os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
+ page)
+
+ # Additionally clean up the diffoscope results default pages
+ log.info('Removing old pages from ' + suite + '/' + arch +
+ '/diffoscope-results/.')
+ try:
+ presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
+ arch + '/diffoscope-results'))
+ except OSError as e:
+ if e.errno != errno.ENOENT: # that's 'No such file or
+ raise # directory' error (errno 17)
+ presents = []
+ log.debug('diffoscope page presents: ' + str(presents))
+ for page in presents:
+ pkg = page.rsplit('.', 1)[0]
+ if (pkg, suite, arch) not in cur_pkgs:
+ log.info('There is no package named ' + pkg + ' from ' +
+ suite + '/' + arch + '/diffoscope-results in ' +
+ 'the database. Removing old page.')
+ os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
+ 'diffoscope-results/' + page)
def purge_old_pages():
for suite in SUITES:
for arch in ARCHS:
- log.info('Removing old pages from ' + suite + '/' + arch + '.')
- try:
- presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
- arch))
- except OSError as e:
- if e.errno != errno.ENOENT: # that's 'No such file or
- raise # directory' error (errno 17)
- presents = []
- log.debug('page presents: ' + str(presents))
-
- # get the existing packages
- query = (
- "SELECT s.name, s.suite, s.architecture "
- "FROM sources s JOIN distributions d on d.id=s.distribution "
- "WHERE s.suite=:suite AND s.architecture=:arch "
- "AND d.name=:dist"
- )
- cur_pkgs = set([
- (p.name, p.suite, p.architecture) for p in query_db(
- sqlalchemy.text(query), suite=suite, arch=arch, dist=DISTRO)
- ])
-
- for page in presents:
- # When diffoscope results exist for a package, we create a page
- # that displays the diffoscope results by default in the main iframe
- # in this subdirectory. Ignore this directory.
- if page == 'diffoscope-results':
- continue
- pkg = page.rsplit('.', 1)[0]
-
- if (pkg, suite, arch) not in cur_pkgs:
- log.info('There is no package named ' + pkg + ' from ' +
- suite + '/' + arch + ' in the database. ' +
- 'Removing old page.')
- os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
- page)
-
- # Additionally clean up the diffoscope results default pages
- log.info('Removing old pages from ' + suite + '/' + arch +
- '/diffoscope-results/.')
- try:
- presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
- arch + '/diffoscope-results'))
- except OSError as e:
- if e.errno != errno.ENOENT: # that's 'No such file or
- raise # directory' error (errno 17)
- presents = []
- log.debug('diffoscope page presents: ' + str(presents))
- for page in presents:
- pkg = page.rsplit('.', 1)[0]
- if (pkg, suite, arch) not in cur_pkgs:
- log.info('There is no package named ' + pkg + ' from ' +
- suite + '/' + arch + '/diffoscope-results in ' +
- 'the database. Removing old page.')
- os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
- 'diffoscope-results/' + page)
+ purge_old_suite_arch_pages(suite, arch)
+
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -25,7 +25,8 @@ from rblib.utils import print_critical_message
from rblib.models import Package
from reproducible_html_live_status import generate_schedule
from reproducible_html_packages import gen_packages_html
-from reproducible_html_packages import purge_old_pages
+from reproducible_html_packages import purge_old_pages, purge_old_arch_suite_pages
+
"""
@@ -797,7 +798,7 @@ if __name__ == '__main__':
for suite in SUITES:
update_sources(suite, arch)
log.info('Sources for suite %s/%s updated at %s.', suite, arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
- purge_old_pages()
+ purge_old_suite_arch_pages(suite, arch)
query = "SELECT count(*) " + \
"FROM schedule AS p JOIN sources AS s ON s.id=p.package_id " + \
"WHERE s.architecture='{}' AND build_type='ci_build'"
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/f7bd6aa0305d245d445fa59628e1947575fd68c5
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/-/commit/f7bd6aa0305d245d445fa59628e1947575fd68c5
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20230705/add9f9aa/attachment-0001.htm>
More information about the Qa-jenkins-scm
mailing list