[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: reproducible debian: scheduler: flake8-fy
Mattia Rizzolo
gitlab at salsa.debian.org
Thu Dec 13 12:35:23 GMT 2018
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
ae7985f3 by Mattia Rizzolo at 2018-12-13T12:24:54Z
reproducible debian: scheduler: flake8-fy
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
d216ef4f by Mattia Rizzolo at 2018-12-13T12:34:02Z
reproducible: add a job importing the openSUSE status into the database and build some html pages
Thanks to Bernhard for all his help IRL! :)
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
4 changed files:
- bin/reproducible.ini
- + bin/reproducible_opensuse_import_json.py
- bin/reproducible_scheduler.py
- job-cfg/reproducible.yaml
Changes:
=====================================
bin/reproducible.ini
=====================================
@@ -31,6 +31,7 @@ defaultarch = amd64
[opensuse]
distro_root = opensuse
+landing_page = opensuse.html
suites = factory
archs = x86_64
defaultsuite = factory
=====================================
bin/reproducible_opensuse_import_json.py
=====================================
@@ -0,0 +1,109 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2018 Mattia Rizzolo <mattia at mapreri.org>
+#
+# Licensed under GPL-2
+#
+# Depends: python3
+
+import json
+from datetime import datetime
+from urllib.request import urlopen
+from sqlalchemy import select, and_, bindparam
+
+from rblib import conn_db, query_db, db_table
+from rblib.confparse import log
+
+json_url = 'http://rb.zq1.de/compare.factory/reproducible.json'
+
+log.info('Downloading status file…')
+ofile = urlopen(json_url).read()
+ostatus = json.loads(ofile)
+
+
+distributions = db_table('distributions')
+sources = db_table('sources')
+results = db_table('results')
+
+distro_id = query_db(
+ select([distributions.c.id]).where(distributions.c.name == 'opensuse')
+)[0][0]
+
+pkgs = []
+pkgs_b = {}
+
+for pkg in ostatus:
+ p = {
+ 'name': pkg['package'],
+ 'version': pkg['version'],
+ 'suite': 'factory',
+ 'architecture': pkg['architecture'],
+ 'distribution': distro_id
+ }
+ pkgs.append(p)
+ data = {
+ 'status': pkg['status'],
+ 'build_date': datetime.fromtimestamp(pkg['build_date']),
+ 'build_duration': pkg['build_duration'],
+ }
+ pkgs_b[(pkg['package'], pkg['version'])] = data
+
+
+# just insert everything for now
+
+log.info('Dropping old data…')
+transaction = conn_db.begin()
+d = results.delete(results.c.package_id.in_(
+ select([sources.c.id]).select_from(sources).where(sources.c.distribution == distro_id)
+))
+query_db(d)
+d = sources.delete(sources.c.distribution == distro_id)
+query_db(d)
+transaction.commit()
+
+
+log.info('Injecting new source packages…')
+transaction = conn_db.begin()
+conn_db.execute(sources.insert(), pkgs)
+transaction.commit()
+
+
+log.info('Injecting build results…')
+cur_pkgs = select(
+ [sources.c.id, sources.c.name, sources.c.version,
+ sources.c.suite, sources.c.architecture]
+).select_from(
+ sources.join(distributions)
+).where(
+ and_(
+ distributions.c.name == 'opensuse',
+ sources.c.suite == bindparam('suite'),
+ sources.c.architecture == bindparam('arch')
+ )
+)
+cur_pkgs = query_db(cur_pkgs.params({'suite': 'factory', 'arch': 'x86_64'}))
+
+builds = []
+for pkg in cur_pkgs:
+ # (id, name, version, suite, architecture)
+ data = pkgs_b[(pkg[1], pkg[2])]
+ if data['status'] == 'nobinaries':
+ continue
+ pkg_status = data['status'].\
+ replace('unreproducible', 'FTBR').\
+ replace('notforus', 'NFU').\
+ replace('waitdep', 'depwait')
+ p = {
+ 'package_id': pkg[0],
+ 'version': pkg[2],
+ 'status': pkg_status,
+ 'build_date': data['build_date'],
+ 'build_duration': data['build_duration'],
+ 'job': 'external',
+ }
+ builds.append(p)
+if builds:
+ transaction = conn_db.begin()
+ conn_db.execute(results.insert(), builds)
+transaction.commit()
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -15,7 +15,6 @@ import lzma
import deb822
import smtplib
import apt_pkg
-apt_pkg.init_system()
from sqlalchemy import sql
from urllib.request import urlopen
from email.mime.text import MIMEText
@@ -30,6 +29,8 @@ from reproducible_html_live_status import generate_schedule
from reproducible_html_packages import gen_packages_html
from reproducible_html_packages import purge_old_pages
+apt_pkg.init_system()
+
"""
How the scheduler chooses which limit to apply, based on the MAXIMA
and LIMIT arrays:
@@ -68,7 +69,7 @@ LIMITS_E404 defines how many packages with status E404 are rescheduled at max.
"""
# only old packages older than this will be rescheduled
-MINIMUM_AGE = {'amd64': 9, 'i386': 12, 'arm64': 11, 'armhf':25 }
+MINIMUM_AGE = {'amd64': 9, 'i386': 12, 'arm64': 11, 'armhf': 25}
# maximum queue size, see explainations above
MAXIMA = {'amd64': 2000, 'i386': 1600, 'arm64': 2000, 'armhf': 1800}
# limits, see explainations above
@@ -334,7 +335,7 @@ def update_sources_db(suite, arch, sources):
if updated_pkgs:
transaction = conn_db.begin()
update_query = sources_table.update().\
- where(sources_table.c.id == sql.bindparam('update_id'))
+ where(sources_table.c.id == sql.bindparam('update_id'))
conn_db.execute(update_query, updated_pkgs)
transaction.commit()
=====================================
job-cfg/reproducible.yaml
=====================================
@@ -659,6 +659,10 @@
my_timed: '42 23 1 1 *'
my_hname: 'profitbricks4'
my_disabled: true
+ - 'opensuse_import_json':
+ my_description: 'Import the status json'
+ my_timed: 'H H * * H'
+ my_shell: 'cd /srv/jenkins/bin/ && ./reproducible_opensuse_import_json.py && ./reproducible_html_indexes.py --distro opensuse && ./reproducible_html_all_packages.py --distro opensuse'
my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
my_hname: ''
my_timeout: '1440'
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/502138c149494047eb19900b7ac408d64079cbe3...d216ef4f0e1e0015e3a0180680e0d8faee7809f4
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/502138c149494047eb19900b7ac408d64079cbe3...d216ef4f0e1e0015e3a0180680e0d8faee7809f4
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20181213/cfbf94fc/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list