[Blends-commit] [SCM] website branch, master, updated. 1b7f8f139d8fe63426160eb3a2f6ccb6147ae660
Andreas Tille
tille at debian.org
Tue Nov 6 10:32:04 UTC 2012
The following commit has been merged in the master branch:
commit 1b7f8f139d8fe63426160eb3a2f6ccb6147ae660
Author: Andreas Tille <tille at debian.org>
Date: Tue Nov 6 11:31:24 2012 +0100
Move script to create tasks statistics graphs from SVN to Git
diff --git a/misc/team_analysis_tools/README.statistics b/misc/team_analysis_tools/README.statistics
new file mode 100644
index 0000000..e2d282b
--- /dev/null
+++ b/misc/team_analysis_tools/README.statistics
@@ -0,0 +1,4 @@
+debian-med source packages version 0.16 and 0.17 were build
+with a broken cdd-dev (version 0.5.1) which ignored continued
+lines in tasks files. This leads to a lower number of
+Recommends.
diff --git a/misc/team_analysis_tools/count-dependencies.py b/misc/team_analysis_tools/count-dependencies.py
new file mode 100755
index 0000000..614b425
--- /dev/null
+++ b/misc/team_analysis_tools/count-dependencies.py
@@ -0,0 +1,246 @@
+#!/usr/bin/python
+
+"""
+This script extracts numbers of dependencies of different metapackages of a Blend
+"""
+
+from sys import stderr, exit
+from os import listdir
+from os.path import isfile, exists
+from fnmatch import fnmatch
+import re
+from subprocess import Popen, PIPE
+from debian import deb822
+from mx.DateTime import *
+import datetime
+
+from mpl_toolkits.mplot3d import Axes3D
+import matplotlib.pyplot as plt
+import matplotlib.colors as colors
+import matplotlib.cm as cmx
+
+debug=0
+
+# FIXME: Blend name should be parameter - currently only used for Debian Med
+BLEND='med'
+# BLEND='science'
+
+# Some tasks contain only a few dependencies which are just spoiling the 3D view
+# Make sure we are drawing a certain selection and all those tasks that might
+# have more dependencies than these
+DRAWTASKS = { 'med': ['bio', 'bio-dev', 'epi', 'practice', 'psychology'],
+ }
+
+def TasksGetDrawList(tasks):
+ task_last = tasks[sorted(tasks.keys())[-1]]
+ # evaluate minimum number of dependencies to draw
+ min2draw = 100
+ drawlist = []
+ for t in DRAWTASKS[BLEND]:
+ if task_last.nrecommended[t] < min2draw:
+ min2draw = task_last.nrecommended[t]
+ for r in task_last.recommends.keys():
+ if task_last.nrecommended[r] >= min2draw:
+ drawlist.append(r)
+
+ drawlist.sort()
+ return drawlist
+
+
+class taskscontent:
+ def __init__(self, version=None, date=None, htask=None, hrecommended=None):
+ self.version = version
+ self.date = date
+ self.mxdate = DateFrom(date)
+ self.year = self.mxdate.year
+ self.month = self.mxdate.month
+ self.day = self.mxdate.day
+ self.datekey = "%04i%02i%02i" % (self.year, self.month, self.day)
+ self.recommends = {}
+ self.nrecommended = {}
+ if htask != None and hrecommended != None:
+ self.nrecommended[htask] = int(hrecommended)
+ self.recommends[htask] = []
+ self.ctrlfile = None
+
+ def add_recommends(self, package, recommends):
+ p = re.sub(BLEND+'-', '', package.encode('utf-8'))
+
+ if not self.recommends.has_key(p):
+ self.recommends[p] = []
+ for r in recommends.encode('utf-8').split(','):
+ r = r.strip()
+ # since we also need to investigate "Depends" for older metapackages we also
+ # need to exclude the extra control packages
+ if r.startswith(BLEND+'-common') or r.startswith(BLEND+'-config') or r.startswith(BLEND+'-tasks'):
+ continue
+ self.recommends[p].append(r)
+ self.nrecommended[p] = len(self.recommends[p])
+
+ def __str__(self):
+ s="Version: %s; Date: %s-%02i-%02i:" % (self.version, self.year, int(self.month), int(self.day))
+ for r in sorted(self.recommends.keys()):
+ if not self.nrecommended.has_key(r):
+ self.nrecommended[r] = len(self.recommends[r])
+ s += '\n ' + r + ': ' + str(self.nrecommended[r])
+ return s
+
+
+def main():
+ root = '../../../projects/'+BLEND+'/tags'
+ u_dirs = listdir(root)
+
+ tasks = {}
+ tasks_found = []
+
+ # Read some data about first med-* packages which never made it into any Vcs
+ if BLEND == 'med':
+ import json
+ fp=open('med_historical_data.json')
+ mjs=fp.read()
+ medhist = json.loads(mjs)
+ fp.close()
+ for mver in medhist:
+ if not mver.has_key('task'):
+ continue
+ if mver['task'] == 'bio-contrib':
+ # we are not interested in non-free dependencies
+ continue
+ if mver['task'] == 'dent':
+ mver['task'] = 'dental'
+ if not mver['task'] in tasks_found:
+ tasks_found.append(mver['task'])
+ task = taskscontent(mver['version'], mver['date'], mver['task'], mver['recommends'])
+ if tasks.has_key(task.datekey):
+ # try to match several single metapackages to what we know today as multibinary
+ mtask = tasks[task.datekey]
+ if mtask.version != task.version:
+ mtask.version = mtask.version + '+' + task.version
+ if mtask.recommends.has_key(mver['task']):
+ print >>stderr, "Duplicated task at same date."
+ continue
+ mtask.recommends[mver['task']] = []
+ mtask.nrecommended[mver['task']] = int(mver['recommends'])
+ else:
+ tasks[task.datekey] = task
+
+ for u in u_dirs:
+ if u == '.svn':
+ continue
+ if len(u.split('.')) > 2:
+ # print >>stderr, "Minor releases usually add no new Dependencies - version %s ignored." % u
+ continue
+ debiandir= root + '/' + u + '/debian/'
+ changelog = debiandir + 'changelog'
+ if not exists(changelog):
+ # print >>stderr, "Version %s seems to be an old package layout tag which is ignored here" % u
+ continue
+
+ # Read output of dpkg-parsechangelog
+ p = Popen("LC_ALL=C dpkg-parsechangelog -l"+changelog, shell=True, bufsize=4096,
+ stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+ errstring = p.stderr.read()
+ if errstring != '':
+ print >>stderr, "Error parsing changelog of version %s\n %s:" % (u, errstring)
+ for stanza in deb822.Sources.iter_paragraphs(p.stdout):
+ if u != stanza['version']:
+ print >>stderr, "Dir %s does not fit changelog version %s" % (u, stanza['version'])
+ else:
+ task = taskscontent(u, stanza['date'])
+
+ # Try to read debian/control
+ ctrlfile = debiandir + 'control'
+ if isfile(ctrlfile):
+ task.ctrlfile = ctrlfile
+ tasks[task.datekey] = task
+ else:
+ print >>stderr, "Unable to open control file for version %s (%s)" % (u, ctrlfile)
+ try:
+ ctrl = open(task.ctrlfile, 'r')
+ except:
+ print >>stderr, "Unable to open control file for version %s (%s) ... even if file exists" % (task.version, task.ctrlfile)
+ if ctrl:
+ ictrl = deb822.Deb822.iter_paragraphs(ctrl)
+ src = ictrl.next()
+ pkg = ictrl.next()
+ while pkg:
+ if pkg.has_key('package'):
+ package = pkg['package']
+ if package != BLEND+'-common' and package != BLEND+'-tasks' and package != BLEND+'-config':
+ if package == 'med-dent':
+ package = 'med-dental'
+ if not package.replace(BLEND+'-', '') in tasks_found:
+ tasks_found.append(package.replace(BLEND+'-', ''))
+ if pkg.has_key('recommends'):
+ recommends = pkg['recommends']
+ task.add_recommends(package,recommends)
+ # in previous package versions we use Depends rather then Recommends
+ if pkg.has_key('depends'):
+ depends = pkg['depends']
+ task.add_recommends(package,depends)
+ try:
+ pkg = ictrl.next()
+ except:
+ pkg = None
+
+ tasks_found.sort()
+
+ drawlist = TasksGetDrawList(tasks)
+
+ cNorm = colors.Normalize(vmin=0, vmax=len(drawlist))
+ jet = plt.get_cmap('jet')
+ scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
+ cs = []
+ for i in range(len(drawlist)):
+ colorVal = scalarMap.to_rgba(i)
+ cs.append(colorVal)
+
+ xtime = []
+ for t in sorted(tasks.keys()):
+ task = tasks[t]
+ xtime.append(datetime.date(task.year, task.month, task.day))
+ values = []
+ for ti in drawlist:
+ v = []
+ for t in sorted(tasks.keys()):
+ task = tasks[t]
+ if task.nrecommended.has_key(ti):
+ v.append(task.nrecommended[ti])
+ else:
+ # v.append(None)
+ v.append(0)
+ # print ti, v, len(v)
+ values.append(v)
+
+ fig = plt.figure()
+ ax = fig.add_subplot(111, projection='3d')
+ for z in range(len(drawlist)):
+ xs = xtime
+ ys = values[z]
+
+ # You can provide either a single color or an array. To demonstrate this,
+ # the first bar of each set will be colored cyan.
+ color = []
+ csz = cs[z]
+ for i in range(len(xtime)):
+ if ys[i] > 0:
+ color.append(csz)
+ else:
+ color.append((1.0, 1.0, 1.0, 0.0)) # how to make zero dependencies invisible ????
+ # col = [cs[z]] * len(xtime)
+ # setting width of bars:
+ # http://stackoverflow.com/questions/886716/controling-bars-width-in-matplotlib-with-per-month-data
+ ax.bar(xtime, values[z], zs=z, zdir='y', color=color, alpha=0.8, width=90)
+
+ ax.set_yticklabels(drawlist)
+ ax.set_xlabel('')
+ ax.set_ylabel('')
+ ax.set_zlabel('')
+
+ plt.show()
+
+
+if __name__ == '__main__':
+ main()
+
+# vim:set et tabstop=2:
diff --git a/misc/team_analysis_tools/debian-med_1.12.pdf b/misc/team_analysis_tools/debian-med_1.12.pdf
new file mode 100644
index 0000000..e6c5bb8
Binary files /dev/null and b/misc/team_analysis_tools/debian-med_1.12.pdf differ
diff --git a/misc/team_analysis_tools/fake_control b/misc/team_analysis_tools/fake_control
new file mode 100755
index 0000000..dd27a31
--- /dev/null
+++ b/misc/team_analysis_tools/fake_control
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# until version 8.3 of Debian Med metapackages the debian/control file was not
+# part of the source tarball but was rather generated at package build time
+# This turned out to be a mistake for several reasons.
+# To enable reasonable investigation of the packages which were included in the
+# past the control file is recreated using this script from binary files which
+# can be downloaded from snapshots.debian.org
+
+extract_info () {
+ grep -v -e "^Installed-Size: " \
+ -e "^Maintainer: " \
+ -e "^Source: debian-med" \
+ -e "^Version: " \
+ -e "^Section: " \
+ -e "^Priority: " \
+ $1 >> control
+ echo "" >> control
+}
+
+grep -v -e "^Package: " -e "Architecture: " control.000 > control
+
+extract_info control.common
+
+for c in `ls control.[a-z]* | sort` ; do
+ if [ "$c" != "control.common" ] ; then
+ extract_info $c
+ fi
+done
diff --git a/misc/team_analysis_tools/med_historical_data.json b/misc/team_analysis_tools/med_historical_data.json
new file mode 100644
index 0000000..e92279a
--- /dev/null
+++ b/misc/team_analysis_tools/med_historical_data.json
@@ -0,0 +1,53 @@
+[
+ {"README": "This file simplifies the evaluation of historical data of single Debian Med metapackages. Formerly the metapackages were created from single source files and this fake data input file tries to trigger them at the same dates to have a sensible chance for plotting some graphs. The real release dates are added as comments."},
+ {"task": "bio", "version": "0.1-1", "date": "2002-02-06", "recommends": "4"},
+ {"task": "bio-contrib", "version": "0.1-1", "date": "2002-02-06", "recommends": "6"},
+ {"task": "dent", "version": "0.1-1", "date": "2002-02-06", "recommends": "1"},
+ {"comment": "the world has seen the following versions of dent: 0.1-2, 0.2-1, 0.3-1, 0.4-1, 0.5-1 and 0.5-2 - but none of these added additional dependencies"},
+ {"task": "imaging", "version": "0.1-1", "date": "2002-02-06", "recommends": "4"},
+ {"comment": "version 0.1-1 of imaging was actually at date: 2002-04-11 but for simplicity we are faking same date as bio 0.1-1 release"},
+ {"task": "imaging-dev", "version": "0.1-1", "date": "2002-02-06", "recommends": "2"},
+ {"comment": "version 0.1-1 of imaging-dev was actually at date: 2002-04-11 but for simplicity we are faking same date as bio 0.1-1 release"},
+ {"task": "tools", "version": "0.1-1", "date": "2002-02-06", "recommends": "1"},
+ {"task": "bio", "version": "0.1-2", "date": "2002-02-08", "recommends": "6"},
+ {"task": "bio", "version": "0.2-1", "date": "2002-04-14", "recommends": "7"},
+ {"task": "imaging", "version": "0.2-1", "date": "2002-04-14", "recommends": "4"},
+ {"comment": "version 0.2-1 of imaging was actually at date: 2002-05-19 but for simplicity we are faking same date as bio 0.2-1 release"},
+ {"task": "imaging-dev", "version": "0.2-1", "date": "2002-04-14", "recommends": "2"},
+ {"comment": "version 0.2-1 of imaging-dev was actually at date: 2002-05-19 but for simplicity we are faking same date as bio 0.2-1 release"},
+ {"task": "tools", "version": "0.2-1", "date": "2002-04-14", "recommends": "1"},
+ {"comment": "version 0.2-1 of tools was actually at date: 2002-05-19 but for simplicity we are faking same date as bio 0.2-1 release"},
+ {"task": "bio", "version": "0.3-1", "date": "2002-04-19", "recommends": "8"},
+ {"task": "imaging", "version": "0.3-1", "date": "2002-04-19", "recommends": "5"},
+ {"comment": "version 0.3-1 of imaging was actually at date: 2002-05-28 but for simplicity we are faking same date as bio 0.3-1 release"},
+ {"task": "imaging-dev", "version": "0.3-1", "date": "2002-04-19", "recommends": "3"},
+ {"comment": "version 0.3-1 of imaging-dev was actually at date: 2002-06-26 but for simplicity we are faking same date as bio 0.3-1 release"},
+ {"task": "tools", "version": "0.3-1", "date": "2002-04-19", "recommends": "1"},
+ {"comment": "version 0.3-1 of tools was actually at date: 2002-06-25 but for simplicity we are faking same date as bio 0.3-1 release"},
+ {"task": "bio", "version": "0.4-1", "date": "2002-05-18", "recommends": "8"},
+ {"task": "imaging", "version": "0.4-1", "date": "2002-05-18", "recommends": "5"},
+ {"comment": "version 0.4-1 of imaging was actually at date: 2002-06-26 but for simplicity we are faking same date as bio 0.4-1 release"},
+ {"task": "bio", "version": "0.4-2", "date": "2002-05-28", "recommends": "9"},
+ {"task": "imaging", "version": "0.4-2", "date": "2002-05-28", "recommends": "5"},
+ {"comment": "version 0.4-2 of imaging was actually at date: 2002-07-24 but for simplicity we are faking same date as bio 0.4-2 release"},
+ {"comment": "leave out bio 0.4-3 which has no change in number and was released at same day as 0.4-2"},
+ {"task": "bio", "version": "0.5-1", "date": "2002-06-24", "recommends": "9"},
+ {"task": "bio", "version": "0.5-2", "date": "2003-01-05", "recommends": "12"},
+ {"comment": "seems bio 0.5-3 was never released - at least there is no changelog entry"},
+ {"task": "bio", "version": "0.5-4", "date": "2003-04-04", "recommends": "15"},
+ {"task": "imaging", "version": "0.4-3", "date": "2003-04-04", "recommends": "8"},
+ {"comment": "version 0.4-3 of imaging was actually at date: 2003-04-14 but for simplicity we are faking same date as bio 0.5-4 release"},
+ {"task": "imaging-dev", "version": "0.3-2", "date": "2003-04-04", "recommends": "5"},
+ {"comment": "version 0.3-2 of imaging-dev was actually at date: 2003-04-14 but for simplicity we are faking same date as bio 0.5-4 release"},
+ {"task": "tools", "version": "0.3-2", "date": "2003-04-04", "recommends": "2"},
+ {"comment": "version 0.3-2 of tools was actually at date: 2003-04-14 but for simplicity we are faking same date as bio 0.5-4 release"},
+ {"comment": "Drop imaging-dev even if it adds another dependency - plot would become unnecessarily crowded for no extra information", "notask": "imaging-dev", "version": "0.3-3", "date": "2003-04-23", "recommends": "5"},
+ {"task": "bio", "version": "0.5-5", "date": "2003-10-05", "recommends": "17"},
+ {"comment": "version 0.5-5 of bio was actually at date: 2004-01-14 but for simplicity we are faking same date as imaging 0.4-4 release"},
+ {"task": "imaging", "version": "0.4-4", "date": "2003-10-05", "recommends": "7"},
+ {"task": "imaging-dev", "version": "0.3-4", "date": "2003-10-05", "recommends": "4"},
+ {"task": "tools", "version": "0.3-3", "date": "2003-10-05", "recommends": "1"},
+ {"comment": "version 0.3-3 of tools was actually at date: 2003-06-29 but for simplicity we are faking same date as imaging 0.4-4 release"},
+ {"task": "bio", "version": "0.5-6", "date": "2004-02-13", "recommends": "19"},
+ {"task": "cms", "version": "0.1-1", "date": "2004-01-06", "recommends": "1"}
+]
--
Static and dynamic websites for Debian Pure Blends
More information about the Blends-commit
mailing list