[Piuparts-commits] [piuparts] 01/08: detect_{piuparts, network}_issues: report found issues

Wed May 3 13:40:16 UTC 2017

This is an automated email from the git hooks/post-receive script.

holger pushed a commit to branch develop
in repository piuparts.

commit 34d4394d1fee8894f0da4168905e8d64d4be0ab1
Author: Andreas Beckmann <anbe at debian.org>
Date:   Mon May 1 21:57:56 2017 +0200

    detect_{piuparts,network}_issues: report found issues
    
    Signed-off-by: Andreas Beckmann <anbe at debian.org>
    Signed-off-by: Holger Levsen <holger at layer-acht.org>
---
 debian/changelog                     |   5 +-
 master-bin/detect_network_issues.in  |  61 ++++++++++++--------
 master-bin/detect_piuparts_issues.in | 104 +++++++++++++++++++++--------------
 3 files changed, 105 insertions(+), 65 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ab59532..776daec 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -58,9 +58,11 @@ piuparts (0.77) UNRELEASED; urgency=medium
     - Mark logfiles for immediate recycling upon first appearance.
     - Run twice per day, moving new logs to untestable within 24-36 hours.
   * master-bin/detect_network_issues:
-    - Detect outdated reference chroot meta data.
+    - Report the issues found in each logfile.
     - Detect alternate wording of gpg verification error.
   * master-bin/detect_piuparts_issues:
+    - Report the issues found in each logfile.
+    - Detect outdated reference chroot meta data.
     - Recognize more problems caused by a full filesystem.
     - Recognize more failures when connecting to databases.
 
@@ -75,7 +77,6 @@ piuparts (0.77) UNRELEASED; urgency=medium
     - only act on sections with precedence < 100.
   * master-bin/detect_network_issues.in:
     - dynamically report detected issues.
-    - move check for outdated reference chroot to detect_piuparts_issues.
   * master-bin/detect_piuparts_issues.in:
     - dynamically report detected issues.
   * instances/piuparts.conf-template.pejacevic: 
diff --git a/master-bin/detect_network_issues.in b/master-bin/detect_network_issues.in
index 73e44bc..07ef180 100755
--- a/master-bin/detect_network_issues.in
+++ b/master-bin/detect_network_issues.in
@@ -29,29 +29,43 @@ STARTDATE=$(date -u +%s)
 # detect network/mirror problems
 #
 
+newline="
+"
+tab="	"
+PATTERN=
+DISPLAY_PATTERN=
+
+add_pattern()
+{
+	PATTERN="${PATTERN:+${PATTERN}|}($1)"
+	DISPLAY_PATTERN="${DISPLAY_PATTERN:+${DISPLAY_PATTERN}${newline}}${tab}$1"
+}
+
+add_pattern "Cannot initiate the connection to"
+add_pattern "Hash Sum mismatch"
+add_pattern "Failed to fetch.*Could not resolve"
+add_pattern "Failed to fetch.*Something wicked happened resolving"
+add_pattern "Failed to fetch.*Unable to connect to"
+add_pattern "E: Method http has died unexpectedly"
+add_pattern "Some index files failed to download, they have been ignored, or old ones used instead."
+add_pattern "Some index files failed to download. They have been ignored, or old ones used instead."
+add_pattern "W: GPG error: .* Release: The following signatures were invalid: BADSIG"
+add_pattern "W: GPG error: .* InRelease: The following signatures were invalid: NODATA"
+add_pattern "W: GPG error: .* InRelease: Clearsigned file isn't valid, got 'NODATA'"
+add_pattern "E: The repository '.*' does no longer have a Release file."
+add_pattern "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing"
+add_pattern "E: Version '.*' for '.*' was not found"
+add_pattern 'E: Method file has died unexpectedly!'
+add_pattern "E: Sub-process rred received a segmentation fault."
+add_pattern "ERROR: Command failed \(status=-7\):.*'apt-get', 'update'"
+add_pattern 'WARNING: The following packages cannot be authenticated!'
+
 FILE=`mktemp`
 for SECTION in $SECTIONS ; do
 	test -d $MASTER/$SECTION || continue
 	for subdir in fail bugged affected ; do
 		test -d $MASTER/$SECTION/$subdir || continue
-		rgrep -l "Cannot initiate the connection to" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "Hash Sum mismatch" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "Failed to fetch.*Could not resolve" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "Failed to fetch.*Something wicked happened resolving" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "Failed to fetch.*Unable to connect to" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "E: Method http has died unexpectedly" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "Some index files failed to download, they have been ignored, or old ones used instead." $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "Some index files failed to download. They have been ignored, or old ones used instead." $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "W: GPG error: .* Release: The following signatures were invalid: BADSIG" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "W: GPG error: .* InRelease: The following signatures were invalid: NODATA" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "W: GPG error: .* InRelease: Clearsigned file isn't valid, got 'NODATA'" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "E: The repository '.*' does no longer have a Release file." $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "E: Version '.*' for '.*' was not found" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l 'E: Method file has died unexpectedly!' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "E: Sub-process rred received a segmentation fault." $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -E "ERROR: Command failed \(status=-7\):.*'apt-get', 'update'" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l 'WARNING: The following packages cannot be authenticated!' $MASTER/$SECTION/$subdir >> $FILE
+		grep -r -l -E --include '*.log' "$PATTERN" $MASTER/$SECTION/$subdir >> $FILE
 	done
 done
 if [ -s $FILE ] ; then
@@ -61,17 +75,18 @@ if [ -s $FILE ] ; then
 	echo
 	echo "Network problems on detected!"
 	echo "(By grep'ing for"
-	grep rgrep $0 | grep -v "grep rgrep" | cut -d "\$" -f1
+	echo "$DISPLAY_PATTERN"
 	echo "in failed logs.)"
-	echo "Test failures due to modified /usr/share/doc/*/changelog.*"
-	echo "are caused by mirror updates during the test."
 	echo
 	echo "The following logfiles have been deleted:"
 	echo
 	echo "----------------------------------------------------------------------"
 	echo
-	for LOG in $(sort -u $FILE) ; do
-		rm -v $LOG
+	for log in $(sort -u $FILE)
+	do
+		echo "$log"
+		grep -E "$PATTERN" "$log" | perl -e 'print grep !$seen{$_}++, <>;' | head -n 10 | sed "s/^/${tab}/"
+		rm -f "$log"
 	done
 	echo
 fi
diff --git a/master-bin/detect_piuparts_issues.in b/master-bin/detect_piuparts_issues.in
index 621ef3d..14fc359 100755
--- a/master-bin/detect_piuparts_issues.in
+++ b/master-bin/detect_piuparts_issues.in
@@ -29,47 +29,63 @@ STARTDATE=$(date -u +%s)
 # detect piuparts problems
 #
 
+newline="
+"
+tab="	"
+PATTERN=
+DISPLAY_PATTERN=
+
+add_pattern()
+{
+	PATTERN="${PATTERN:+${PATTERN}|}($1)"
+	DISPLAY_PATTERN="${DISPLAY_PATTERN:+${DISPLAY_PATTERN}${newline}}${tab}$1"
+}
+
+add_pattern 'PIUPARTS OUTPUT INCOMPLETE'
+add_pattern "WARNING: History of available packages does not match - reference chroot may be outdated"
+add_pattern "tar( \(child\))?: .*.tar.gz: Cannot open: No such file or directory"
+add_pattern "tar: Error is not recoverable: exiting now"
+add_pattern "gzip: stdin: invalid compressed data--crc error"
+add_pattern "ERROR:.*:Temporary directory is not a directory"
+add_pattern "E: Sub-process Popen returned an error code \(2\)"
+add_pattern "chroot: failed to run command '.*': Permission denied"
+add_pattern "ERROR: Command failed \(status=-7\): .* 'apt-cache'"
+add_pattern 'update-binfmts: warning: unable to close /proc/sys/fs/binfmt_misc/register: Invalid argument'
+add_pattern 'userdel: user .* is currently logged in'
+# Disk space issues.
+add_pattern "No space left on device"
+add_pattern "E: You don't have enough free space in /var/cache/apt/archives/"
+add_pattern "E: Failed to write temporary StateFile /var/lib/apt/extended_states.tmp"
+add_pattern "tar: .*: Wrote only .* of .* bytes"
+add_pattern "passwd: Authentication token manipulation error"
+add_pattern "(chfn|groupadd|useradd): (cannot lock|failure while writing changes to) /etc/(group|gshadow|passwd|shadow|subuid)"
+add_pattern "(groupadd|useradd): (unable to lock|cannot rewrite) (group|password) file"
+# Database server issues.
+add_pattern 'invoke-rc.d: initscript mysql, action "(start|restart)" failed.'
+add_pattern 'invoke-rc.d: initscript postgresql, action "start" failed.'
+add_pattern 'The PostgreSQL server failed to start.'
+add_pattern 'No database found online on port 5432'
+add_pattern 'unable to connect to postgresql server'
+add_pattern 'Failed to connect to postgres.'
+add_pattern 'PG::ConnectionBad: could not connect to server'
+add_pattern 'psql: FATAL:  password authentication failed for user'
+add_pattern 'createuser: could not connect to database postgres: could not connect to server: No such file or directory'
+add_pattern 'Non-default port in /etc/postgresql/.*/main/postgresql.conf'
+add_pattern 'Firebird .* server already running.'
+# These are python errors from piuparts itself. Python errors from maintainer
+# scripts etc. would be indented by 2 spaces, so don't match here.
+add_pattern "^RuntimeError: maximum recursion depth exceeded while calling a Python object"
+add_pattern "^Traceback"
+add_pattern "^OSError:"
+add_pattern "^IndentationError:"
+add_pattern "^SyntaxError:"
+
 FILE=`mktemp`
 for SECTION in $SECTIONS ; do
 	test -d $MASTER/$SECTION || continue
 	for subdir in fail bugged affected ; do
 		test -d $MASTER/$SECTION/$subdir || continue
-		rgrep -l 'PIUPARTS OUTPUT INCOMPLETE' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l "WARNING: History of available packages does not match - reference chroot may be outdated" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -E "tar( \(child\))?: .*.tar.gz: Cannot open: No such file or directory" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "tar: Error is not recoverable: exiting now" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "tar: .*: Wrote only .* of .* bytes" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "gzip: stdin: invalid compressed data--crc error" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "ERROR:.*:Temporary directory is not a directory" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -i -e "No space left on device" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "E: You don't have enough free space in /var/cache/apt/archives/" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "E: Failed to write temporary StateFile /var/lib/apt/extended_states.tmp" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "passwd: Authentication token manipulation error" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -E "(chfn|groupadd|useradd): (cannot lock|failure while writing changes to) /etc/(group|gshadow|passwd|shadow|subuid)" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -E "(groupadd|useradd): (unable to lock|cannot rewrite) (group|password) file" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "E: Sub-process Popen returned an error code (2)" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "chroot: failed to run command '.*': Permission denied" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "ERROR: Command failed (status=-7): .* 'apt-cache'" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "^RuntimeError: maximum recursion depth exceeded while calling a Python object" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "^Traceback" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "^OSError:" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "^IndentationError:" $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e "^SyntaxError:" $MASTER/$SECTION/$subdir >> $FILE
-		# Python errors from maintainer scripts etc. would be indented by 2 spaces, so don't match here.
-		rgrep -l -e 'update-binfmts: warning: unable to close /proc/sys/fs/binfmt_misc/register: Invalid argument' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'userdel: user .* is currently logged in' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -i -E 'invoke-rc.d: initscript mysql, action "(start|restart)" failed.' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -i -e 'invoke-rc.d: initscript postgresql, action "start" failed.' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'The PostgreSQL server failed to start.' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -i -e 'FATAL:  could not create shared memory segment: No space left on device' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'No database found online on port 5432' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'unable to connect to postgresql server' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'Failed to connect to postgres.' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'PG::ConnectionBad: could not connect to server' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'psql: FATAL:  password authentication failed for user' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'createuser: could not connect to database postgres: could not connect to server: No such file or directory' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'Non-default port in /etc/postgresql/.*/main/postgresql.conf' $MASTER/$SECTION/$subdir >> $FILE
-		rgrep -l -e 'Firebird .* server already running.' $MASTER/$SECTION/$subdir >> $FILE
+		grep -r -l -E --include '*.log' "$PATTERN" $MASTER/$SECTION/$subdir >> $FILE
 		find $MASTER/$SECTION/$subdir -name '*.log' -size 0 >> $FILE
 	done
 done
@@ -78,18 +94,26 @@ if [ -s $FILE ] ; then
 	RUNTIME=$(date -u -d "0 $FINALDATE seconds - $STARTDATE seconds" +%T)
 	echo "Runtime: $RUNTIME"
 	echo
-	echo "piuparts problem detected!"
-	echo "(By grep'ing for)"
-	grep rgrep $0 | grep -v "grep rgrep" | cut -d "\$" -f1
+	echo "piuparts problems detected!"
+	echo "(By grep'ing for"
+	echo "$DISPLAY_PATTERN"
 	echo "in failed logs.)"
 	echo
 	echo "Please review the following logfiles/packages."
 	echo "If it is always the same package failing, it's likely to be an issue in the"
 	echo "package."
 	echo
-		sort -u $FILE | xargs rm -v
+	echo "The following logfiles have been deleted:"
 	echo
 	echo "----------------------------------------------------------------------"
 	echo
+	for log in $(sort -u $FILE)
+	do
+		echo "$log"
+		test -s "$log" || echo "${tab}Empty logfile."
+		grep -E "$PATTERN" "$log" | perl -e 'print grep !$seen{$_}++, <>;' | head -n 10 | sed "s/^/${tab}/"
+		rm -f "$log"
+	done
+	echo
 fi
 rm $FILE

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/piuparts/piuparts.git

