[Qa-jenkins-scm] [Git][qa/jenkins.debian.net][master] 2 commits: rblib: pep8-f
Mattia Rizzolo
gitlab at salsa.debian.org
Mon Dec 2 15:34:36 GMT 2019
Mattia Rizzolo pushed to branch master at Debian QA / jenkins.debian.net
Commits:
5e8e1d78 by Mattia Rizzolo at 2019-12-02T15:30:20Z
rblib: pep8-f
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
86160814 by Mattia Rizzolo at 2019-12-02T15:32:58Z
reproducible: add a 'build_type' field to the DB, to support both doule-rebuilds and verification rebuilds
Signed-off-by: Mattia Rizzolo <mattia at debian.org>
- - - - -
12 changed files:
- bin/rblib/html.py
- bin/reproducible_alpine_manual_scheduler.sh
- bin/reproducible_alpine_scheduler.sh
- bin/reproducible_archlinux_manual_scheduler.sh
- bin/reproducible_archlinux_scheduler.sh
- bin/reproducible_build.sh
- bin/reproducible_build_alpine_pkg.sh
- bin/reproducible_build_archlinux_pkg.sh
- bin/reproducible_common.sh
- bin/reproducible_db_maintenance.py
- bin/reproducible_remote_scheduler.py
- bin/reproducible_scheduler.py
Changes:
=====================================
bin/rblib/html.py
=====================================
@@ -25,7 +25,7 @@ tab = ' '
# take a SHA1 of the css page for style version
_hasher = hashlib.sha1()
with open(REPRODUCIBLE_STYLES, 'rb') as f:
- _hasher.update(f.read())
+ _hasher.update(f.read())
REPRODUCIBLE_STYLE_SHA1 = _hasher.hexdigest()
# Templates used for creating package pages
@@ -73,8 +73,8 @@ def _gen_suite_arch_nav_context(suite, arch, suite_arch_nav_template=None,
's': s,
'class': 'current' if s == suite else '',
'uri': _renderer.render(suite_arch_nav_template,
- {'distro': conf_distro['distro_root'],
- 'suite': s, 'arch': arch})
+ {'distro': conf_distro['distro_root'],
+ 'suite': s, 'arch': arch})
if include_suite else '',
})
@@ -85,8 +85,8 @@ def _gen_suite_arch_nav_context(suite, arch, suite_arch_nav_template=None,
'a': a,
'class': 'current' if a == arch else '',
'uri': _renderer.render(suite_arch_nav_template,
- {'distro': conf_distro['distro_root'],
- 'suite': suite, 'arch': a}),
+ {'distro': conf_distro['distro_root'],
+ 'suite': suite, 'arch': a}),
})
return (suite_list, arch_list)
@@ -96,8 +96,9 @@ def create_main_navigation(suite=defaultsuite, arch=defaultarch,
displayed_page=None, suite_arch_nav_template=None,
ignore_experimental=False, no_suite=None,
no_arch=None):
- suite_list, arch_list = _gen_suite_arch_nav_context(suite, arch,
- suite_arch_nav_template, ignore_experimental, no_suite, no_arch)
+ suite_list, arch_list = _gen_suite_arch_nav_context(
+ suite, arch, suite_arch_nav_template,
+ ignore_experimental, no_suite, no_arch)
context = {
'suite': suite,
'arch': arch,
=====================================
bin/reproducible_alpine_manual_scheduler.sh
=====================================
@@ -36,9 +36,9 @@ for PKG in $PACKAGES ; do
echo "Now trying to reschedule $PKG in $SUITE."
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
if [ ! -z "${PKG_ID}" ] ; then
- SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+ SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
if [ -z "$SCHEDULED" ] ; then
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
SUCCESS="$SUCCESS $PKG"
else
echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
=====================================
bin/reproducible_alpine_scheduler.sh
=====================================
@@ -47,7 +47,7 @@ update_alpine_repositories() {
done
TOTAL=$(cat ${ALPINE_PKGS}_full_pkgbase_list | wc -l)
echo "$(date -u ) - $TOTAL alpine packages are now known to alpine."
- local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
echo "$(date -u) - updating alpine repositories, currently $total packages scheduled."
#
@@ -109,7 +109,7 @@ update_alpine_repositories() {
echo "new package found: $REPO/$pkgbase $version "
query_db "INSERT into sources (name, version, suite, architecture, distribution) VALUES ('$PKG', '$version', '$SUITE', '$ARCH', $DISTROID);"
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
elif [ "$VERSION" != "$version" ] ; then
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- /sbin/apk version -t $version $VERSION || true)"
if [ "$VERCMP" = ">" ] ; then
@@ -129,10 +129,10 @@ update_alpine_repositories() {
echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
- SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+ SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
if [ -z "$SCHEDULED" ] ; then
- echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ echo " INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
else
echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
fi
@@ -170,12 +170,12 @@ update_alpine_repositories() {
AND s.architecture='x86_64'
AND (r.status LIKE 'DEPWAIT%' OR r.status LIKE '404%')
AND r.build_date < '$MINDATE'
- AND s.id NOT IN (SELECT package_id FROM schedule)
+ AND s.id NOT IN (SELECT package_id FROM schedule WHERE build_type = 'ci_build')
LIMIT $MAX;"
local DEPWAIT404=$(query_db "$QUERY")
if [ ! -z "$DEPWAIT404" ] ; then
for PKG_ID in $DEPWAIT404 ; do
- QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+ QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
query_db "$QUERY"
done
echo "$(date -u ) - done scheduling $(echo -n "$DEPWAIT404" | wc -l ) packages in DEPWAIT_ or 404_ states."
@@ -192,7 +192,7 @@ update_alpine_repositories() {
local THRESHOLD=600
MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "4 days ago")
SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
- local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
if [ $CURRENT -le $THRESHOLD ] ; then
echo "$(date -u ) - scheduling $MAX old packages."
QUERY="SELECT s.id, s.name, max(r.build_date) max_date
@@ -201,13 +201,13 @@ update_alpine_repositories() {
AND s.architecture='x86_64'
AND r.status != 'blacklisted'
AND r.build_date < '$MINDATE'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule WHERE build_type = 'ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT $MAX;"
local OLD=$(query_db "$QUERY")
for PKG_ID in $(echo -n "$OLD" | cut -d '|' -f1) ; do
- QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+ QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
query_db "$QUERY"
done
echo "$(date -u ) - done scheduling $MAX old packages."
@@ -219,7 +219,7 @@ update_alpine_repositories() {
# output stats
#
rm "$ALPINE_PKGS"_full_pkgbase_list
- total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type = 'ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
new=$(cat $NEW | wc -l 2>/dev/null|| true)
updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
old=$(echo -n "$OLD" | wc -l 2>/dev/null|| true)
=====================================
bin/reproducible_archlinux_manual_scheduler.sh
=====================================
@@ -35,9 +35,9 @@ for PKG in $PACKAGES ; do
echo "Now trying to reschedule $PKG in $SUITE."
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
if [ ! -z "${PKG_ID}" ] ; then
- SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+ SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
if [ -z "$SCHEDULED" ] ; then
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
SUCCESS="$SUCCESS $PKG"
else
echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
=====================================
bin/reproducible_archlinux_scheduler.sh
=====================================
@@ -47,7 +47,7 @@ update_archlinux_repositories() {
done | sort -u -R > "$ARCHLINUX_PKGS"_full_pkgbase_list
TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages are now known to Arch Linux."
- local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type = 'ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
echo "$(date -u) - updating Arch Linux repositories, currently $total packages scheduled."
#
@@ -109,7 +109,7 @@ update_archlinux_repositories() {
echo "new package found: $REPO/$pkgbase $version "
query_db "INSERT into sources (name, version, suite, architecture, distribution) VALUES ('$PKG', '$version', '$SUITE', '$ARCH', $DISTROID);"
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
elif [ "$VERSION" != "$version" ] ; then
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
if [ "$VERCMP" = "1" ] ; then
@@ -128,11 +128,11 @@ update_archlinux_repositories() {
echo $REPO/$pkgbase >> $UPDATED
echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
- echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
- SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+ echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID} AND build_type='ci_build';"
+ SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type='ci_build';")
if [ -z "$SCHEDULED" ] ; then
- echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
- query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+ echo " INSERT INTO schedule (package_id, date_scheduled build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
+ query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
else
echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
fi
@@ -170,12 +170,12 @@ update_archlinux_repositories() {
AND s.architecture='x86_64'
AND (r.status LIKE 'DEPWAIT%' OR r.status LIKE '404%')
AND r.build_date < '$MINDATE'
- AND s.id NOT IN (SELECT package_id FROM schedule)
+ AND s.id NOT IN (SELECT package_id FROM schedule WHERE build_type='ci_build')
LIMIT $MAX;"
local DEPWAIT404=$(query_db "$QUERY")
if [ ! -z "$DEPWAIT404" ] ; then
for PKG_ID in $DEPWAIT404 ; do
- QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+ QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
query_db "$QUERY"
done
echo "$(date -u ) - done scheduling $(echo -n "$DEPWAIT404" | wc -l ) packages in DEPWAIT_ or 404_ states."
@@ -192,7 +192,7 @@ update_archlinux_repositories() {
local THRESHOLD=600
MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "4 days ago")
SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
- local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
if [ $CURRENT -le $THRESHOLD ] ; then
echo "$(date -u ) - scheduling $MAX old packages."
QUERY="SELECT s.id, s.name, max(r.build_date) max_date
@@ -201,13 +201,13 @@ update_archlinux_repositories() {
AND s.architecture='x86_64'
AND r.status != 'blacklisted'
AND r.build_date < '$MINDATE'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule WHERE build_type='ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT $MAX;"
local OLD=$(query_db "$QUERY")
for PKG_ID in $(echo -n "$OLD" | cut -d '|' -f1) ; do
- QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+ QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) 'ALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
query_db "$QUERY"
done
echo "$(date -u ) - done scheduling $MAX old packages."
@@ -219,7 +219,7 @@ update_archlinux_repositories() {
# output stats
#
rm "$ARCHLINUX_PKGS"_full_pkgbase_list
- total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+ total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
new=$(cat $NEW | wc -l 2>/dev/null|| true)
updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
old=$(echo -n "$OLD" | wc -l 2>/dev/null|| true)
=====================================
bin/reproducible_build.sh
=====================================
@@ -137,7 +137,7 @@ update_db_and_html() {
query_db "INSERT INTO results (package_id, version, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPKGID', '$VERSION', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB') ON CONFLICT (package_id) DO UPDATE SET version='$VERSION', status='$STATUS', build_date='$DATE', build_duration='$DURATION', node1='$NODE1', node2='$NODE2', job='$JOB' WHERE results.package_id='$SRCPKGID'"
query_db "INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPACKAGE', '$VERSION', '$SUITE', '$ARCH', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')"
# unmark build since it's properly finished
- query_db "DELETE FROM schedule WHERE package_id='$SRCPKGID';"
+ query_db "DELETE FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build';"
gen_package_html $SRCPACKAGE
echo
echo "$(date -u) - successfully updated the database and updated $DEBIAN_URL/rb-pkg/${SUITE}/${ARCH}/$SRCPACKAGE.html"
@@ -413,6 +413,7 @@ choose_package() {
SELECT s.suite, s.id, s.name, s.version, sch.save_artifacts, sch.notify, s.notify_maintainer, sch.date_scheduled
FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
WHERE sch.date_build_started is NULL
+ AND sch.build_type='ci_build'
AND s.distribution=$DISTROID
AND s.architecture='$ARCH'
ORDER BY date_scheduled LIMIT 5"|sort -R|head -1)
@@ -442,12 +443,12 @@ choose_package() {
rm -f $BAD_BUILDS
# mark build attempt, first test if none else marked a build attempt recently
echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "ok, $SRCPACKAGE is not building anywhere…"
# try to update the schedule with our build attempt, then check no else did it, if so, abort
- query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+ query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND build_type='ci_build' AND date_build_started IS NULL"
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
handle_race_condition
=====================================
bin/reproducible_build_alpine_pkg.sh
=====================================
@@ -53,7 +53,7 @@ update_pkg_in_db() {
echo "$QUERY"
query_db "$QUERY"
# unmark build since it's properly finished
- QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}';"
+ QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}' AND build_type='ci_build';"
echo "$QUERY"
query_db "$QUERY"
rm pkg.build_duration pkg.state pkg.version
@@ -68,6 +68,7 @@ choose_package() {
FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
WHERE distribution=$DISTROID
AND sch.date_build_started is NULL
+ AND sch.build_type='ci_build'
AND s.architecture='$ARCH'
ORDER BY date_scheduled LIMIT 10"|sort -R|head -1)
if [ -z "$RESULT" ] ; then
@@ -94,12 +95,12 @@ choose_package() {
rm -f $BAD_BUILDS
# mark build attempt, first test if none else marked a build attempt recently
echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "ok, $SRCPACKAGE is not building anywhere…"
# try to update the schedule with our build attempt, then check no else did it, if so, abort
- query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+ query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL AND build_type='ci_build'"
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
handle_race_condition
=====================================
bin/reproducible_build_archlinux_pkg.sh
=====================================
@@ -53,7 +53,7 @@ update_pkg_in_db() {
echo "$QUERY"
query_db "$QUERY"
# unmark build since it's properly finished
- QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}';"
+ QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}' and build_type='ci_build';"
echo "$QUERY"
query_db "$QUERY"
rm pkg.build_duration pkg.state pkg.version
@@ -68,6 +68,7 @@ choose_package() {
FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
WHERE distribution=$DISTROID
AND sch.date_build_started is NULL
+ AND sch.build_type='ci_build'
AND s.architecture='$ARCH'
ORDER BY date_scheduled LIMIT 10"|sort -R|head -1)
if [ -z "$RESULT" ] ; then
@@ -94,12 +95,12 @@ choose_package() {
rm -f $BAD_BUILDS
# mark build attempt, first test if none else marked a build attempt recently
echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "ok, $SRCPACKAGE is not building anywhere…"
# try to update the schedule with our build attempt, then check no else did it, if so, abort
- query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
- RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+ query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND build_type='ci_build' AND date_build_started IS NULL"
+ RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
if [ -z "$RESULT" ] ; then
echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
handle_race_condition
=====================================
bin/reproducible_common.sh
=====================================
@@ -712,7 +712,7 @@ cleanup_pkg_files() {
}
handle_race_condition() {
- local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
+ local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'") # XXX not only ci_build
local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
log_warning "$msg"
printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
@@ -727,7 +727,7 @@ handle_race_condition() {
unregister_build() {
# unregister this build so it will immeditiatly tried again
if [ -n "$SRCPKGID" ] ; then
- query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID"
+ query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID AND build_type='ci_build'" # XXX not only ci_build
fi
NOTIFY=""
}
=====================================
bin/reproducible_db_maintenance.py
=====================================
@@ -718,6 +718,14 @@ schema_updates = {
"UPDATE stats_build SET status='reproducible' WHERE status='GOOD'",
"UPDATE stats_build SET status='blacklisted' WHERE status='BLACKLISTED'"
],
+ 46: [ # crete a build_type field
+ "CREATE TYPE build_type AS ENUM ('verification', 'ci_build')",
+ "ALTER TABLE results ADD COLUMN build_type build_type DEFAULT 'ci_build' NOT NULL",
+ "ALTER TABLE stats_build ADD COLUMN build_type build_type DEFAULT 'ci_build' NOT NULL",
+ "ALTER TABLE schedule ADD COLUMN build_type build_type",
+ "UPDATE schedule SET build_type='ci_build'",
+ "ALTER TABLE schedule ALTER COLUMN build_type SET NOT NULL",
+ ],
}
=====================================
bin/reproducible_remote_scheduler.py
=====================================
@@ -217,7 +217,7 @@ def rest(scheduling_args, requester, local, suite, arch):
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
- WHERE p.package_id='{id}'"""
+ WHERE p.package_id='{id}' AND p.build_type='ci_build'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
@@ -316,6 +316,7 @@ def rest(scheduling_args, requester, local, suite, arch):
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
+ 'build_type': 'ci_build',
})
else:
add_to_schedule.append({
@@ -324,6 +325,7 @@ def rest(scheduling_args, requester, local, suite, arch):
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
+ 'build_type': 'ci_build',
})
save_schedule.append({
=====================================
bin/reproducible_scheduler.py
=====================================
@@ -13,11 +13,9 @@
import sys
import lzma
import deb822
-import smtplib
import apt_pkg
from sqlalchemy import sql
from urllib.request import urlopen
-from email.mime.text import MIMEText
from datetime import datetime, timedelta
from rblib import query_db, db_table
@@ -449,7 +447,7 @@ def queue_packages(all_pkgs, packages, date):
def schedule_packages(packages):
- pkgs = [{'package_id': x, 'date_scheduled': packages[x]} for x in packages.keys()]
+ pkgs = [{'package_id': x, 'date_scheduled': packages[x], 'build_type': 'ci_build'} for x in packages.keys()]
log.debug('IDs about to be scheduled: %s', packages.keys())
if pkgs:
conn_db.execute(db_table('schedule').insert(), pkgs)
@@ -469,7 +467,7 @@ def query_untested_packages(suite, arch, limit):
SELECT sources.id, sources.name FROM sources
WHERE sources.suite='{suite}' AND sources.architecture='{arch}'
AND sources.id NOT IN
- (SELECT schedule.package_id FROM schedule)
+ (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
AND sources.id NOT IN
(SELECT results.package_id FROM results)
ORDER BY random()
@@ -488,7 +486,7 @@ def query_new_versions(suite, arch, limit):
AND s.version != r.version
AND r.status != 'blacklisted'
AND s.id IN (SELECT package_id FROM results)
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
GROUP BY s.id, s.name, s.version, r.version
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
@@ -513,7 +511,7 @@ def query_old_ftbfs_versions(suite, arch, limit):
AND r.status='FTBFS'
AND ( n.bugs = '[]' OR n.bugs IS NULL )
AND r.build_date < '{date}'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -532,7 +530,7 @@ def query_old_depwait_versions(suite, arch, limit):
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status='depwait'
AND r.build_date < '{date}'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -552,7 +550,7 @@ def query_old_versions(suite, arch, limit):
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status != 'blacklisted'
AND r.build_date < '{date}'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch,
@@ -570,7 +568,7 @@ def query_e404_versions(suite, arch, limit):
WHERE s.suite='{suite}' AND s.architecture='{arch}'
AND r.status = 'E404'
AND r.build_date < '{date}'
- AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+ AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
GROUP BY s.id, s.name
ORDER BY max_date
LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -698,7 +696,7 @@ def schedule_e404_versions(arch, total):
def scheduler(arch):
query = "SELECT count(*) " + \
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " + \
- "WHERE s.architecture='{arch}'"
+ "WHERE s.architecture='{arch}' AND build_type='ci_build'"
total = int(query_db(query.format(arch=arch))[0][0])
log.info('==============================================================')
log.info('Currently scheduled packages in all suites on ' + arch + ': ' + str(total))
@@ -734,7 +732,7 @@ def scheduler(arch):
for suite in priotized_suite_order:
query = "SELECT count(*) " \
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " \
- "WHERE s.suite='{suite}' AND s.architecture='{arch}'"
+ "WHERE s.suite='{suite}' AND s.architecture='{arch}' AND p.build_type='ci_build'"
query = query.format(suite=suite, arch=arch)
now_queued_here[suite] = int(query_db(query)[0][0]) + \
len(untested[suite]+new[suite]+old[suite])
@@ -783,7 +781,7 @@ if __name__ == '__main__':
purge_old_pages()
query = "SELECT count(*) " + \
"FROM schedule AS p JOIN sources AS s ON s.id=p.package_id " + \
- "WHERE s.architecture='{}'"
+ "WHERE s.architecture='{}' AND build_type='ci_build'"
message = ''
for arch in ARCHS:
log.info('Scheduling for %s...', arch)
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6d8111ce5a342ff1b6c715034032badbba64cce9...86160814a97111a61cbba3e9a3604b7d93053370
--
View it on GitLab: https://salsa.debian.org/qa/jenkins.debian.net/compare/6d8111ce5a342ff1b6c715034032badbba64cce9...86160814a97111a61cbba3e9a3604b7d93053370
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/qa-jenkins-scm/attachments/20191202/f15b92a0/attachment-0001.html>
More information about the Qa-jenkins-scm
mailing list