[Piuparts-devel] [PATCH 3/3] Use the bug tracking system to analyze bugs.
debian at mikapflueger.de
debian at mikapflueger.de
Fri Jul 29 13:41:13 UTC 2011
From: Mika Pflüger <mika at mikapflueger.de>
piuparts-analyze.py: Rewrite to use python-debianbts to analyze if bugs are filed already.
debian/control: Add python-apt and python-debianbts to piuparts depends.
Also rewrite the heuristics to catch new versions of packages with old errors.
---
debian/control | 2 +-
piuparts-analyze.py | 122 ++++++++++++++++++++++++++++++++++++++-------------
2 files changed, 92 insertions(+), 32 deletions(-)
diff --git a/debian/control b/debian/control
index 62c8715..69d54f1 100644
--- a/debian/control
+++ b/debian/control
@@ -14,7 +14,7 @@ X-Python-Version: >= 2.6
Package: piuparts
Architecture: all
Depends: apt, ${python:Depends}, debootstrap, lsof, lsb-release,
- python-debian, ${misc:Depends}
+ python-debian, ${misc:Depends}, python-debianbts, python-apt
Suggests: python-rpy, ghostscript
Description: .deb package installation, upgrading, and removal testing tool
piuparts tests that .deb packages (as used by Debian) handle
diff --git a/piuparts-analyze.py b/piuparts-analyze.py
index 3896c64..9664e2d 100644
--- a/piuparts-analyze.py
+++ b/piuparts-analyze.py
@@ -1,6 +1,8 @@
#!/usr/bin/python
+# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw at iki.fi)
+# Copyright 2011 Mika Pflüger (debian at mikapflueger.de)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
@@ -19,11 +21,12 @@
"""Analyze failed piuparts logs and move them around when the errors are known.
-This program looks at piuparts log files in ./fail, and compares them to
-log files in ./bugged. If it finds one in ./fail that has the same
-error messages as one in ./bugged, it copies the headers to the one in ./fail
-and moves it to ./bugged. This is useful for repetitive uploads of the
-same package that do not fix the problem.
+This program looks at piuparts log files in ./fail, and queries the bts to find
+out if bugs have been filed already. If so, it moves them to ./bugged.
+It tries to detect if new versions of bugged packages are uploaded without solving
+the bug and will then update the bts with the new found versions, and copy the
+headers of the log in ./fail to the one in ./bugged and vice versa. It will then
+move the failed log to ./bugged as well.
"""
@@ -31,20 +34,20 @@ same package that do not fix the problem.
import os
import re
import shutil
-import sys
+import subprocess
+import debianbts
+import apt_pkg
+
+apt_pkg.init_system()
error_pattern = re.compile(r"(?<=\n).*error.*\n?", flags=re.IGNORECASE)
chroot_pattern = re.compile(r"tmp/tmp.*?'")
-def find_logs(dir):
- return [os.path.join(dir, x)
- for x in os.listdir(dir) if x.endswith(".log")]
-
-
-def package_name(log):
- return os.path.basename(log).split("_", 1)[0]
+def find_logs(directory):
+ return [os.path.join(directory, x)
+ for x in os.listdir(directory) if x.endswith(".log")]
def find_bugged_logs(failed_log):
@@ -53,6 +56,14 @@ def find_bugged_logs(failed_log):
return [x for x in find_logs("bugged") if pat in x]
+def package_name(log):
+ return os.path.basename(log).split("_", 1)[0]
+
+
+def package_version(log):
+ return os.path.basename(log).split("_", 1)[1].rstrip('.log')
+
+
def extract_errors(log):
"""This pretty stupid implementation is basically just 'grep -i error', and then
removing the timestamps and the name of the chroot and the package version itself."""
@@ -90,7 +101,6 @@ def prepend_to_file(filename, data):
f = file(filename, "r")
old_data = f.read()
f.close()
-
f = file(filename + ".tmp", "w")
f.write(data)
f.write(old_data)
@@ -103,27 +113,65 @@ def prepend_to_file(filename, data):
os.remove(filename + "~")
-def mark_bugged(failed_log, bugged_log):
- print "Moving", failed_log, "to bugged"
+def get_bug_versions(bug):
+ """Gets a list of only the version numbers for which the bug is found."""
+ # debianbts returns it in the format package/1.2.3 which will become 1.2.3
+ return [x.split('/', 1)[1] for x in debianbts.get_status((bug,))[0].found_versions]
+
+
+def move_to_bugged(failed_log):
+ print("Moving %s to bugged" % failed_log)
+ os.rename(failed_log, os.path.join("bugged", os.path.basename(failed_log)))
+
+
+def mark_bugged_version(failed_log, bugged_log):
+ """Copies the headers from the old log to the new log and vice versa and
+ moves the new log to bugged. Removes the old log in bugged."""
+ bugged_headers = extract_headers(bugged_log)
+ failed_headers = extract_headers(failed_log)
+ prepend_to_file(failed_log, bugged_headers)
+ prepend_to_file(bugged_log, failed_headers)
+ move_to_bugged(failed_log)
- headers = extract_headers(bugged_log)
- prepend_to_file(failed_log, headers)
- if os.path.isdir(".bzr"):
- assert "'" not in failed_log
- os.system("bzr move '%s' bugged" % failed_log)
- else:
- os.rename(failed_log,
- os.path.join("bugged", os.path.basename(failed_log)))
+def bts_update_found(bugnr, newversion):
+ # Disabled for now as I'm not sure automatic bug updating is wanted
+ #subprocess.check_call(('bts', 'found', bugnr, newversion))
+ print(' '.join(('bts', 'found', str(bugnr), newversion)))
-def mark_logs_with_known_bugs():
+
+def mark_logs_with_reported_bugs():
for failed_log in find_logs("fail"):
+ pname = package_name(failed_log)
+ pversion = package_version(failed_log)
failed_errors = extract_errors(failed_log)
- for bugged_log in find_bugged_logs(failed_log):
- bugged_errors = extract_errors(bugged_log)
- if failed_errors == bugged_errors:
- mark_bugged(failed_log, bugged_log)
- break
+ moved = False
+ for bug in piuparts_bugs_in(pname):
+ for bug_version in get_bug_versions(bug):
+
+ if apt_pkg.version_compare(pversion, bug_version) == 0: # pversion == bug_version
+ if not moved:
+ move_to_bugged(failed_log)
+ moved = True
+ break
+
+ elif apt_pkg.version_compare(pversion, bug_version) > 0: # pversion > bug_version
+ bugged_logs = find_bugged_logs(failed_log)
+ if not bugged_logs:
+ print('%s/%s: Maybe the bug was filed earlier: %d against %s/%s'
+ % (pname, pversion, bug, pname, bug_version))
+ for bugged_log in bugged_logs:
+ old_pversion = package_version(bugged_log)
+ bugged_errors = extract_errors(bugged_log)
+ if (apt_pkg.version_compare(old_pversion, bug_version) == 0 # old_pversion == bug_version
+ and
+ failed_errors == bugged_errors):
+ # a bug was filed for an old version of the package,
+ # and the errors were the same back then - assume it is the same bug.
+ if not moved:
+ mark_bugged_version(failed_log, bugged_log)
+ moved = True
+ bts_update_found(bug, pversion)
def report_packages_with_many_logs():
@@ -149,8 +197,20 @@ def report_packages_with_many_logs():
print
+piuparts_usertags_cache = None
+def all_piuparts_bugs():
+ global piuparts_usertags_cache
+ if piuparts_usertags_cache is None:
+ piuparts_usertags_cache = debianbts.get_usertag("debian-qa at lists.debian.org", 'piuparts')['piuparts']
+ return piuparts_usertags_cache
+
+
+def piuparts_bugs_in(package):
+ return debianbts.get_bugs('package', package, 'bugs', all_piuparts_bugs())
+
+
def main():
- mark_logs_with_known_bugs()
+ mark_logs_with_reported_bugs()
report_packages_with_many_logs()
--
1.7.5.4
More information about the Piuparts-devel
mailing list