[Qa-jenkins-scm] [jenkins.debian.net] 01/01: new script and job to find obsolete transitional packages in (currently) buster
Holger Levsen
holger at layer-acht.org
Mon Oct 23 12:23:12 UTC 2017
This is an automated email from the git hooks/post-receive script.
holger pushed a commit to branch master
in repository jenkins.debian.net.
commit 2e98abc7a2eae041cc2cb947810d4f912924e903
Author: Holger Levsen <holger at layer-acht.org>
Date: Mon Oct 23 14:23:03 2017 +0200
new script and job to find obsolete transitional packages in (currently) buster
Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
bin/find_obsolete_transitional_packages.sh | 190 +++++++++++++++++++++++++++++
job-cfg/obsolete-transitional.yaml | 73 +++++++++++
logparse/obsolete-transitional.rules | 17 +++
3 files changed, 280 insertions(+)
diff --git a/bin/find_obsolete_transitional_packages.sh b/bin/find_obsolete_transitional_packages.sh
new file mode 100755
index 0000000..fb1fd41
--- /dev/null
+++ b/bin/find_obsolete_transitional_packages.sh
@@ -0,0 +1,190 @@
+#!/bin/bash
+
+# Copyright 2017 Holger Levsen <holger at layer-acht.org>
+# released under the GPLv=2
+
+#
+# run with any parameter for interactive mode which will fire up mutt for 10 buggy packages
+# else automatic mode is assumed, which has more output
+#
+
+DEBUG=false
+if [ -f /srv/jenkins/bin/common-functions.sh ] ; then
+ . /srv/jenkins/bin/common-functions.sh
+ common_init "$@"
+else
+ #normally defined in common-functions.sh
+ export MIRROR=http://deb.debian.org/debian
+ #for quicker development:
+ #PACKAGES[0]=/home/schroots/jessie/var/lib/apt/lists/deb.debian.org_debian_dists_jessie_main_binary-amd64_Packages
+ #PACKAGES[1]=/var/lib/apt/lists/deb.debian.org_debian_dists_stretch_main_binary-amd64_Packages
+ #PACKAGES[2]=/home/schroots/sid/var/lib/apt/lists/deb.debian.org_debian_dists_sid_main_binary-amd64_Packages
+fi
+
+if ! which chdist ; then
+ echo "Please install devscripts."
+ exit 1
+elif ! which grep-dctrl ; then
+ echo "Please install grep-dctrl."
+ exit 1
+fi
+
+if [ -z "$1" ] ; then
+ echo "Call $(basename $0) with 2 or 3 params:"
+ echo " 1st: suite which is being developed, eg 'buster'"
+ echo " 2nd: suites to look at, eg. 'jenkins stretch sid'"
+ echo " 3rd: any third param will enable manual mode."
+ exit 0
+fi
+
+LANG="en_EN.UTF-8"
+ARCH=amd64
+SUITES="$2" # jessie stretch sid
+OLDSTABLE="jessie"
+STABLE="stretch"
+NEXT="$1" # buster
+if [ "$NEXT" != "buster" ] ; then
+ echo "This script needs more changes to work on other suites than buster…"
+ echo "Not many, but a very few."
+ exit 1
+fi
+# transitional packages we know bugs have been filed about…
+BUGGED="multiarch-support jadetex dh-systemd libpcap-dev transfig myspell-it myspell-sl python-gobject ttf-dejavu ttf-dejavu-core ttf-dejavu-extra libav-tools netcat gnupg2 libkf5akonadicore-bin qml-module-org-kde-extensionplugin myspell-ca myspell-en-gb myspell-sv-se myspell-lt khelpcenter4 libqca2-plugin-ossl gambas3-gb-desktop-gnome git-core gperf-ace libalberta2-dev asterisk-prompt-it libatk-adaptor-data kdemultimedia-dev kdemultimedia-kio-plugins autoconf-gl-macros autofs5 autofs5-h [...]
+
+BASEPATH=$(ls -1d /tmp/transitional-????? 2>/dev/null || true)
+if [ -z "$BASEPATH" ] ; then
+ BASEPATH=$(mktemp -t $TMPDIR -d transitional-XXXXX)
+
+ for SUITE in $SUITES ; do
+ mkdir -p $BASEPATH/$SUITE
+ # the "[arch=$ARCH]" is a workaround until #774685 is fixed
+ chdist --data-dir=$BASEPATH/$SUITE --arch=$ARCH create $SUITE-$ARCH "[arch=$ARCH]" $MIRROR $SUITE main
+ # in interactive mode we don't care about sources
+ if [ -n "$3" ] ; then
+ sed -i "s#deb-src#\#deb-src#g" $BASEPATH/$SUITE/$SUITE-$ARCH/etc/apt/sources.list
+ fi
+ chdist --data-dir=$BASEPATH/$SUITE --arch=$ARCH apt-get $SUITE-$ARCH update
+ echo
+ done
+fi
+
+NR=0
+for SUITE in $SUITES ; do
+ PACKAGES[$NR]=$(ls $BASEPATH/$SUITE/$SUITE-$ARCH/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages)
+ echo "PACKAGES[$NR] = ${PACKAGES[$NR]}"
+ # only in interactive mode we care about sources
+ if [ -z "$3" ] ; then
+ SOURCES[$NR]=$(ls $BASEPATH/$SUITE/$SUITE-$ARCH/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources)
+ echo "SOURCES[$NR] = ${SOURCES[$NR]}"
+ fi
+ echo
+ let NR=$NR+1
+done
+
+
+BAD=""
+BAD_COUNTER=0
+GOOD_COUNTER=0
+BUGGED_COUNTER=0 ; for i in $BUGGED ; do let BUGGED_COUNTER=$BUGGED_COUNTER+1 ; done
+for PKG in $(grep-dctrl -sPackage -n -FDescription "transitional.*package" --ignore-case --regex ${PACKAGES[1]}) ; do
+ if [ "${PKG:0:9}" = "iceweasel" ] || [ "${PKG:0:7}" = "icedove" ] || [ "${PKG:0:6}" = "iceowl" ] || [ "${PKG:0:9}" = "lightning" ]; then
+ echo "ignore iceweasel, icedove, iceowl, lightning and friends…: $PKG"
+ continue
+ fi
+ if echo " $BUGGED " | egrep -q " $PKG " ; then
+ echo "ignore $PKG because a bug has already been filed."
+ continue
+ fi
+ OLDSTABLE_HIT=$(grep-dctrl -sPackage -n -FDescription "transitional.*package" --ignore-case --regex ${PACKAGES[0]} |egrep "^$PKG$")
+ if [ -z "$OLDSTABLE_HIT" ] ; then
+ echo "$PKG not in $OLDSTABLE, so new transitional package, so fine."
+ else
+ SID_HIT=$(grep-dctrl -sPackage -n -FDescription "transitional.*package" --ignore-case --regex ${PACKAGES[2]} |egrep "^$PKG$")
+ if [ -z "$SID_HIT" ] ; then
+ echo "Transitional package $PKG in $OLDSTABLE and $STABLE, but not in sid, yay!"
+ let GOOD_COUNTER=$GOOD_COUNTER+1
+ else
+ let BAD_COUNTER=$BAD_COUNTER+1
+ echo "SIGH #$BAD_COUNTER: $PKG is a transitional package in $OLDSTABLE, $STABLE and sid. Someone should file a bug."
+ BAD="$BAD $PKG"
+ fi
+ fi
+done
+echo
+
+# interactive mode
+if [ -n "$3" ] ; then
+ MAX=20
+ NR=0
+ echo "Entering manual mode, filing $MAX bugs."
+ for PKG in $BAD ; do
+ echo "firefox https://packages.debian.org/$PKG ;"
+ let NR=$NR+1
+ if [ $NR -eq $MAX ] ; then
+ echo "Filed $MAX bugs, ending."
+ break
+ fi
+ done
+ echo "Please open those firefox tabs… and press enter"
+ read a
+ NR=0
+ for PKG in $BAD ; do
+ SRC=$(grep-dctrl -sSource -FPackage -n $PKG --exact-match ${PACKAGES[2]} | cut -d " " -f1)
+ if [ -z "$SRC" ] ; then
+ SRC=$PKG
+ fi
+ VERSION=$(grep-dctrl -sVersion -FPackage -n $PKG --exact-match ${PACKAGES[1]})
+ VERBOSE=$( ( for SAUCE in ${PACKAGES[0]} ${PACKAGES[1]} ${PACKAGES[2]} ; do
+ grep-dctrl -sPackage,Description,Version -FPackage $PKG --exact-match $SAUCE
+ done ) | sort -u)
+ #firefox https://packages.debian.org/$PKG &
+ TMPFILE=`mktemp`
+ cat >> $TMPFILE <<- EOF
+Package: $PKG
+Version: $VERSION
+Severity: normal
+user: qa.debian.org at packages.debian.org
+usertags: transitional
+
+Please drop the transitional package $PKG for $NEXT,
+as it has been released with $OLDSTABLE and $STABLE already.
+
+$VERBOSE
+
+Thanks for maintaining $SRC!
+
+EOF
+ mutt -s "please drop transitional package $PKG" -i $TMPFILE submit at bugs.debian.org
+ rm $TMPFILE
+
+ let NR=$NR+1
+ if [ $NR -eq $MAX ] ; then
+ echo "Filed $MAX bugs, ending."
+ break
+ fi
+ done
+
+else
+ # non-interactive mode
+ for PKG in $BAD ; do
+ echo
+ ( for SAUCE in ${PACKAGES[0]} ${PACKAGES[1]} ${PACKAGES[2]} ; do
+ grep-dctrl -sPackage,Description,Version -FPackage $PKG --exact-match $SAUCE
+ done ) | sort -u
+ done
+ echo
+ echo $BAD | dd-list --sources ${SOURCES[2]} -i
+fi
+
+echo
+echo "Found $BAD_COUNTER bad packages (=transitional dummy package in $OLDSTABLE, $STABLE and sid) and $GOOD_COUNTER removed transitional packages (=doesn't exist in sid) plus we know about $BUGGED_COUNTER open bugs about obsolete transitional packages."
+
+echo "In the future, this script should probably complain about transitional packages in stretch and buster, and suggest to file wishlist bugs for those. Though probably it's more useful to file wishlist bugs against packages depending on those, first (or do both)… and should those latter be normal severity?"
+
+echo
+if [ "${BASEPATH:0:5}" = "/tmp/" ] ; then
+ rm $BASEPATH -r
+else
+ du -sch $BASEPATH
+ echo "please rm $BASEPATH manually."
+fi
diff --git a/job-cfg/obsolete-transitional.yaml b/job-cfg/obsolete-transitional.yaml
new file mode 100644
index 0000000..96f4863
--- /dev/null
+++ b/job-cfg/obsolete-transitional.yaml
@@ -0,0 +1,73 @@
+- defaults:
+ name: obsolete-transitional
+ project-type: freestyle
+ properties:
+ - sidebar:
+ url: https://jenkins.debian.net/userContent/about.html
+ text: About jenkins.debian.net
+ icon: /userContent/images/debian-swirl-24x24.png
+ - sidebar:
+ url: https://jenkins.debian.net/view/haskell/
+ text: Haskell packages metadata consistency
+ icon: /userContent/images/debian-jenkins-24x24.png
+ - sidebar:
+ url: http://www.profitbricks.co.uk
+ text: Sponsored by Profitbricks
+ icon: /userContent/images/profitbricks-24x24.png
+ description: '{my_description}{do_not_edit}'
+ logrotate:
+ daysToKeep: 365
+ numToKeep: 365
+ artifactDaysToKeep: -1
+ artifactNumToKeep: -1
+ builders:
+ - shell: '/srv/jenkins/bin/bin/find_obsolete_transitional_packages.sh {my_distro} "{my_suites}"'
+ publishers:
+ - logparser:
+ parse-rules: '/srv/jenkins/logparse/obsolete-transitional.rules'
+ unstable-on-warning: 'false'
+ fail-on-error: 'false'
+ - email:
+ recipients: '{my_recipients}'
+ #- plot:
+ # - title: Stats
+ # yaxis: Packages
+ # group: PlotGroup
+ # csv-file-name: package-plan-stats.csv
+ # series:
+ # - file: stats.csv
+ # format: csv
+ # # inclusion-flag: exclude-by-string
+ # # exclude: total
+ # - title: Totals
+ # yaxis: Packages
+ # group: PlotGroup
+ # style: area
+ # series:
+ # - file: stats.csv
+ # format: csv
+ # # inclusion-flag: include-by-string
+ # # exclude: total
+ - naginator:
+ progressive-delay-increment: 5
+ progressive-delay-maximum: 15
+ max-failed-builds: 5
+ regular-expression: 'Caused by: hudson.plugins.git.GitException: Command "git fetch'
+
+- job-template:
+ defaults: obsolete-transitional
+ name: '{name}'
+ triggers:
+ - pollscm:
+ cron: '*/6 * * * *'
+ - timed: '0 16 * * *' # every day at 16:00 UTC
+
+- project:
+ name: obsolete-transitional
+ do_not_edit: '<br>Find obsolete transitional packages.'
+ jobs:
+ - '{name}_buster':
+ my_distro: 'buster'
+ my_suites: 'jessie stretch sid'
+ my_description: 'Find obsolete transitional packages in buster.'
+ my_recipients: 'jenkins+debian-qa qa-jenkins-scm at lists.alioth.debian.org'
diff --git a/logparse/obsolete-transitional.rules b/logparse/obsolete-transitional.rules
new file mode 100644
index 0000000..8fa9b8a
--- /dev/null
+++ b/logparse/obsolete-transitional.rules
@@ -0,0 +1,17 @@
+# see https://wiki.jenkins-ci.org/display/JENKINS/Log+Parser+Plugin
+
+ok /not really/
+
+# suppress warnings from commit messages
+ok /HEAD is now at .+/
+
+# match line starting with 'error ', case-insensitive
+error /(?i)^error /
+
+# list of warnings here...
+warning /[Ww]arning/
+warning /WARNING/
+
+# create a quick access link to lines in the report containing 'INFO'
+info /INFO/
+
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/qa/jenkins.debian.net.git
More information about the Qa-jenkins-scm
mailing list