[Piuparts-commits] rev 343 - trunk
Holger Levsen
holger at alioth.debian.org
Fri Apr 17 13:37:25 UTC 2009
Author: holger
Date: 2009-04-17 13:37:25 +0000 (Fri, 17 Apr 2009)
New Revision: 343
Modified:
trunk/README.txt
trunk/TODO
trunk/piuparts-report.py
Log:
create source centric packages pages
Modified: trunk/README.txt
===================================================================
--- trunk/README.txt 2009-04-17 10:31:02 UTC (rev 342)
+++ trunk/README.txt 2009-04-17 13:37:25 UTC (rev 343)
@@ -353,7 +353,7 @@
These settings are used for all sections. Except for the first two they are all mandatory:
-* "sections" defaults to sid and defines which sections should be processed in master-slave mode. Each section defined here has to have a section with the section specific settings explained below.
+* "sections" defaults to sid and defines which sections should be processed in master-slave mode. Each section defined here has to have a section with the section specific settings explained below. The first section defined should always be sid, because the data from first section a package is in is used for the source package html report.
* "idle-sleep" is the length of time the slave should wait before querying the master again if the master didn't have any new packages to test. In seconds, so a value of 300 would mean five minutes, and that seems to be a good value when there are fairly few slaves per master. The default is 10 seconds.
Modified: trunk/TODO
===================================================================
--- trunk/TODO 2009-04-17 10:31:02 UTC (rev 342)
+++ trunk/TODO 2009-04-17 13:37:25 UTC (rev 343)
@@ -4,17 +4,19 @@
must for 0.36
- report:
---- write_sources_summaries() should be split in prepare_ and assemble_. assemble_ should use find_log() to build all needed pages
--- find_log() is not used atm
---- include pts links
+--- refactor code, sources.txt should not be calculated where it currently is
--- include links to http://qa.debian.org/developer.php?login=$maintainer_email
--- include links to logfiles in source summary pages
+--- include links to source pages in state summary pages
--- dont write counts.txt if it has already been written that day
-- slave
--- mention distro when sending logs
--- slave_run: cleanup stale proc mountpoints
--- cronjob: warn about 10 or more tmp filesystems...
+--- slave_stop script - unneeded with patch for #523958
- then R
+- probably include patch for #523950
- take care of old conf files on upgrades
- take care of old pyc files from python-central
- test pipuparts with piuparts before uploading
Modified: trunk/piuparts-report.py
===================================================================
--- trunk/piuparts-report.py 2009-04-17 10:31:02 UTC (rev 342)
+++ trunk/piuparts-report.py 2009-04-17 13:37:25 UTC (rev 343)
@@ -284,15 +284,7 @@
Package $package
</td>
</tr>
- <tr class="normalrow">
- $sourcedata
- </tr>
- <tr class="titlerow">
- <td class="titlecell" colspan="3">
- Binary package(s) in $section
- </td>
- </tr>
- $binaryrows
+ $rows
</table>
"""
@@ -442,16 +434,22 @@
return str
-def path_to_source_summary_page(source):
+def source_subdir(source):
if source[:3] == "lib":
return source[:4]
else:
return source[:1]
-def find_log_files(dir):
- return [name for name in os.listdir(dir) if name.endswith(".log")]
+def find_files_with_suffix(dir,suffix):
+ name=[name for name in os.listdir(dir) if name.endswith(suffix)]
+ subdirs=os.listdir(dir)
+ for subdir in subdirs:
+ if os.path.isdir(os.path.join(dir,subdir)):
+ name+=[name for name in os.listdir(os.path.join(dir,subdir)) if name.endswith(suffix)]
+ return name
+
def update_file(source, target):
if os.path.exists(target):
aa = os.stat(source)
@@ -471,7 +469,6 @@
target = os.path.join(fulldir, basename)
update_file(source, target)
-
def remove_old_logs(logs_by_dir, output_dir):
for dir in logs_by_dir:
fulldir = os.path.join(output_dir, dir)
@@ -492,7 +489,6 @@
f.write(contents)
f.close()
-
def create_section_navigation(section_names):
tablerows = ""
for section in section_names:
@@ -500,7 +496,43 @@
(html_protect(section), html_protect(section))
return tablerows;
+def merge_templates(templates, section_names, source_pages_output_dir):
+ logging.debug("Merging package templates from %s" % section_names)
+ for src_tpl in templates:
+ source_tpl = os.path.join(source_pages_output_dir,source_subdir(src_tpl),src_tpl)
+ f = file(source_tpl, "r")
+ rows = file.read(f)
+ f.close()
+ os.unlink(source_tpl)
+ template_path = source_tpl[:-len(".tpl_src")]
+ for section in section_names:
+ section_tpl = os.path.join(template_path+".tpl_"+section)
+ if os.path.isfile(section_tpl):
+ f = file(section_tpl, "r")
+ rows += file.read(f)
+ f.close()
+ os.unlink(section_tpl)
+
+ htmlpage = string.Template(HTML_HEADER + SOURCE_PACKAGE_BODY_TEMPLATE + HTML_FOOTER)
+ package = os.path.basename(src_tpl)[:-len(".tpl_src")]
+ filename = os.path.join(source_pages_output_dir,source_subdir(package),package+".html")
+ f = file(filename, "w")
+ f.write(htmlpage.safe_substitute( {
+ "section_navigation": create_section_navigation(section_names),
+ "time": time.strftime("%Y-%m-%d %H:%M %Z"),
+ "rows": rows,
+ }))
+ f.close()
+
+def assemble_source_pages(output_directory,section_names):
+ logging.debug("Assembling all source package summaries in %s" % output_directory)
+
+ # needs to find them in subdir...
+ templates = find_files_with_suffix(output_directory, ".tpl_src")
+ merge_templates(templates, section_names, output_directory)
+
+
class Section:
def __init__(self, section):
@@ -576,7 +608,7 @@
dirs = ["pass", "fail", "bugged", "fixed", "reserved", "untestable"]
logs_by_dir = {}
for dir in dirs:
- logs_by_dir[dir] = find_log_files(dir)
+ logs_by_dir[dir] = find_files_with_suffix(dir, ".log")
logging.debug("Copying log files")
copy_logs(logs_by_dir, self._output_directory)
@@ -615,13 +647,14 @@
logging.debug("Writing page for %s" % state)
list = "<ul>\n"
for package in self._binary_db.get_packages_in_state(state):
- list += "<li>%s (%s)" % (html_protect(package["Package"]),
+ list += "<li>%s (%s)" % (
+ html_protect(package["Package"]),
html_protect(package["Maintainer"]))
if package.dependencies():
list += "\n<ul>\n"
for dep in package.dependencies():
list += "<li>dependency %s is %s</li>\n" % \
- (html_protect(dep),
+ (html_protect(dep),
emphasize_reason(html_protect(self._binary_db.state_by_name(dep))))
list += "</ul>\n"
list += "</li>\n"
@@ -636,7 +669,6 @@
}))
def write_counts_summary(self):
-
logging.debug("Writing counts.txt")
header = "date"
counts = "%s" % time.strftime("%Y%m%d")
@@ -661,49 +693,54 @@
return nn
return None
- def write_sources_summaries(self):
- logging.debug("Writing source package summaries")
+ def prepare_package_summaries(self):
+ logging.debug("Writing package templates in %s" % self._config.section)
+
sources = ""
for source in self._source_db.get_all_packages():
- binaries = self._source_db.get_control_header(source, "Binary")
- maintainer = self._source_db.get_control_header(source, "Maintainer")
- version = self._source_db.get_control_header(source, "Version")
- success = True
- failed = False
- sourcedata = "<td class=\"contentcell2\">%s</td><td class=\"contentcell2\" colspan=\"2\">%s</td>" % (html_protect(source), html_protect(maintainer))
- binaryrows = ""
- for binary in binaries.split(", "):
- state = self._binary_db.state_by_name(binary)
- binaryrows += "<tr class=\"normalrow\"><td class=\"contentcell2\">%s</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\">%s</td></tr>" % (binary, state, version)
- if state != "successfully-tested":
- success = False
- if state == "failed-testing":
- failed = True
- source_state="unknown"
- if success: source_state="success"
- if failed: source_state="failed"
- sources += "%s: %s\n" % (source, source_state)
- summary_page_path = os.path.join(self._output_directory, "source", path_to_source_summary_page(source))
- if not os.path.exists(summary_page_path):
- os.makedirs(summary_page_path)
- filename = os.path.join(summary_page_path, (source + ".html"))
- htmlpage = string.Template(HTML_HEADER + SOURCE_PACKAGE_BODY_TEMPLATE + HTML_FOOTER)
+
+ summary_page_path = os.path.join(self._output_directory, "../source", source_subdir(source))
+ if not os.path.exists(summary_page_path):
+ os.makedirs(summary_page_path)
+
+ binaries = self._source_db.get_control_header(source, "Binary")
+ version = self._source_db.get_control_header(source, "Version")
+ maintainer = self._source_db.get_control_header(source, "Maintainer")
+
+ sourcerows = "<tr class=\"normalrow\"><td class=\"contentcell2\"><a href=\"http://packages.qa.debian.org/%s\" target=\"_blank\">%s</a></td><td class=\"contentcell2\" colspan=\"2\">%s</td></tr>" % (source, html_protect(source), html_protect(maintainer))
+
+ filename = os.path.join(summary_page_path, (source + ".tpl_src"))
+ if not os.path.isfile(filename):
f = file(filename, "w")
- f.write(htmlpage.safe_substitute( {
- "section_navigation": create_section_navigation(self._section_names),
- "time": time.strftime("%Y-%m-%d %H:%M %Z"),
- "package": html_protect(source),
- "sourcedata": sourcedata,
- "section": html_protect(self._config.section),
- "binaryrows": binaryrows,
- }))
+ f.write(sourcerows)
f.close()
+
+ success = True
+ failed = False
+ binaryrows = "<tr class=\"titlerow\"><td class=\"titlecell\" colspan=\"3\">Binary package(s) in "+self._config.section+"</td></tr>"
+ for binary in binaries.split(", "):
+ state = self._binary_db.state_by_name(binary)
+ binaryrows += "<tr class=\"normalrow\"><td class=\"contentcell2\">%s</td><td class=\"contentcell2\">%s</td><td class=\"contentcell2\">%s</td></tr>" % (binary, state, version)
+ if state != "successfully-tested":
+ success = False
+ if state == "failed-testing":
+ failed = True
+ filename = os.path.join(summary_page_path, (source + ".tpl_"+self._config.section))
+ f = file(filename, "w")
+ f.write(binaryrows)
+ f.close()
+
+ source_state="unknown"
+ if success: source_state="success"
+ if failed: source_state="failed"
+ sources += "%s: %s\n" % (source, source_state)
+
write_file(os.path.join(self._output_directory, "sources.txt"), sources)
def generate_file_output(self):
- self.write_counts_summary()
- if self._config["sources-url"]:
- self.write_sources_summaries()
+ self.write_counts_summary()
+ if self._config["sources-url"]:
+ self.prepare_package_summaries()
def generate_output(self, master_directory, output_directory, section_names):
self._section_names = section_names
@@ -721,8 +758,9 @@
self.generate_file_output()
os.chdir(oldcwd)
+ else:
+ logging.debug("Warning: %s does not exist. Did you ever let the slave work on %s?" % self._master_directory, self._config.section)
-
def main():
setup_logging(logging.DEBUG, None)
@@ -743,6 +781,8 @@
section.generate_output(master_directory=global_config["master-directory"],output_directory=global_config["output-directory"],section_names=section_names)
sections.append(section)
+ assemble_source_pages(os.path.join(global_config["output-directory"],"source"),section_names)
+
logging.debug("Writing index page")
htmlpage = string.Template(HTML_HEADER + INDEX_BODY_TEMPLATE + HTML_FOOTER)
write_file(os.path.join(global_config["output-directory"],"index.html"), htmlpage.safe_substitute( {
More information about the Piuparts-commits
mailing list