[Blends-commit] [SCM] blends-gsoc branch, master, updated. e1544514f1b0fc5620a6ed5347379e70f481b326
Emmanouil Kiagias
e.kiagias at gmail.com
Tue Jul 16 18:22:04 UTC 2013
The following commit has been merged in the master branch:
commit e1544514f1b0fc5620a6ed5347379e70f481b326
Author: Emmanouil Kiagias <e.kiagias at gmail.com>
Date: Tue Jul 16 20:21:45 2013 +0200
a first working instance of tasks_diff script. First it can dump a json file containing the dependencies of a release, and then using this file can compare the json-stored dependencies with another(new) release
diff --git a/tasks_diff b/tasks_diff
new file mode 100755
index 0000000..3ef52f2
--- /dev/null
+++ b/tasks_diff
@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+
+# Copyright 2013: Emmanouil Kiagias <e.kiagias at gmail.com>
+# License: GPL
+
+"""
+this module is still under construction, just testing stuff around
+before I write the complete script
+"""
+
+import os
+import re
+import sys
+import json
+import pprint
+import logging
+import argparse
+from debian import deb822
+
+def clean_up_packages(packages):
+ logger = logging.getLogger(__name__)
+ # Hack: Debian Edu tasks files are using '\' at EOL which is broken
+ # in RFC 822 files, but blend-gen-control from blends-dev relies
+ # on this. So remove this stuff here for the Moment
+ pkgs = re.sub('\\\\\n\s+', '', packages)
+
+ # Remove versions from versioned depends
+ pkgs = re.sub(' *\([ ><=\.0-9]+\) *', '', pkgs)
+
+ # temporary strip spaces from alternatives ('|') to enable erroneous space handling as it was done before
+ pkgs = re.sub('\s*\|\s*', '|', pkgs)
+
+ # turn alternatives ('|') into real depends for this purpose
+ # because we are finally interested in all alternatives
+ pkgslist = pkgs.split(',')
+ # Collect all dependencies in one line first,
+ # create an object for each later
+ pkgs_in_one_line = []
+ for depl in pkgslist:
+ dl = depl.strip()
+ if dl != '': # avoid confusion when ',' is at end of line
+ if re.search('\s', dl):
+ logger.error("Blend %s task %s: Syntax error '%s'" % (blend, task, dl))
+ # trying to fix the syntax error after issuing error message
+ dlspaces = re.sub('\s+', ',', dl).split(',')
+ for dls in dlspaces:
+ pkgs_in_one_line.append(dls.strip())
+ logger.info("Blend %s task %s: Found '%s' package inside broken syntax string - please fix task file anyway" % (blend, task, dls.strip()))
+ else:
+ # in case we have to deal with a set of alternatives
+ if re.search('\|', dl):
+ #for da in dl.split('|'):
+ # deps_in_one_line.append(da)
+ dl = re.sub('\|', ' | ', dl)
+ pkgs_in_one_line.append(dl)
+ # self.inject_package_alternatives(blend, task, strength, dl)
+
+ return pkgs_in_one_line
+
+def load_task(path_to_task):
+ """
+ parses a task file and return a dictionary containing all its package headers elements
+ (depends, suggests etc)
+ """
+ ftask = open(path_to_task, 'r')
+ task = os.path.basename(path_to_task)
+ taskinfo = {}
+
+ for header in ["depends", "suggests", "recommends", "ignore", "avoid"]:
+ taskinfo[header] = []
+
+ for paragraph in deb822.Sources.iter_paragraphs(ftask, shared_storage=False):
+ if paragraph.has_key("depends"):
+ taskinfo["depends"] += clean_up_packages(paragraph["depends"])
+
+ if paragraph.has_key("suggests"):
+ taskinfo["suggests"] += clean_up_packages(paragraph["suggests"])
+
+ if paragraph.has_key("recommends"):
+ taskinfo["recommends"] += clean_up_packages(paragraph["recommends"])
+
+ if paragraph.has_key("ignore"):
+ taskinfo["ignore"] += clean_up_packages(paragraph["ignore"])
+
+ if paragraph.has_key("avoid"):
+ taskinfo["avoid"] += clean_up_packages(paragraph["avoid"])
+
+ return task, taskinfo
+
+def compare_tasks(tasks, tasks_compare):
+ """
+ This function will dump in stdout the package differences between
+ the given tasks1 and tasks2
+ """
+
+ first_print = True
+
+ for task in tasks:
+ if not task in tasks_compare:
+ continue
+
+ task_first = True
+ first_add = True
+ for header in ["depends", "recommends", "suggests", "ignore", "avoid"]:
+ added = set(tasks_compare[task][header]) - set(tasks[task][header])
+ if added:
+ if first_print:
+ print "* Changes in metapackage dependencies"
+ first_print = False
+ if task_first:
+ print " -{0}".format(task)
+ task_first = False
+ if first_add:
+ print " added:"
+ first_add = False
+ print " {0}: ".format(header.capitalize()),
+ print ", ".join(added)
+
+ first_remove = True
+ for header in ["depends", "recommends", "suggests", "ignore", "avoid"]:
+ removed = set(tasks[task][header]) - set(tasks_compare[task][header])
+ if removed:
+ if first_print:
+ print "* Changes in metapackage dependencies"
+ first_print = False
+ if task_first:
+ print " -{0}".format(task)
+ task_first = False
+ if first_remove:
+ print " removed:"
+ first_remove = False
+ print " {0}: ".format(header.capitalize()),
+ print ", ".join(removed)
+
+
+ removed_tasks = set(tasks.keys()) - set(tasks_compare.keys())
+ added_tasks = set(tasks_compare.keys()) - set(tasks.keys())
+ if added_tasks:
+ print "* New metapackages:"
+ for newtask in added_tasks:
+ print " -{0}".format(newtask)
+
+ if removed_tasks:
+ print "* Removed metapackages:"
+ for removedtask in removed_tasks:
+ print " -{0}".format(removedtask)
+
+
+def load_tasks(tasks_path):
+ tasks = {}
+
+ for taskpath in tasks_path:
+ taskname, taskinfo = load_task(taskpath)
+ tasks[taskname] = taskinfo
+
+ return tasks
+
+if __name__ == "__main__":
+ default_json = "tasks.json"
+
+ parser = argparse.ArgumentParser(epilog="Example: ./tasks_diff.py -t med/tags/1.13/ -s -d;\
+ ./tasks_diff.py -t med/tags/1.13.1/ -c -d")
+
+ parser.add_argument("-t", "--tasks", dest="tasks", type=str,
+ help="Path to task files", default=".")
+ parser.add_argument("-s", "--status-dump", dest="statusdump", action="store_true",
+ help="Dump dependencies status into a json file")
+ parser.add_argument("-o", "--output", dest="output", type=str, default=default_json,
+ help="Output file where to store the dependencies json file(when -s/--statusdump is provided)")
+ parser.add_argument("-j", "--json-file", dest="jsonfile", type=str, default=default_json,
+ help="Path to json file containing a previous release dependencies. \
+ This json file will be compared with the task files in the provided tasks directory(-t/--tasks argument) \
+ if the -c/--compare argument is provided. Default json file to be used(if no argument is provided): {0}".format(default_json))
+ parser.add_argument("-c", "--compare", dest="compare", action="store_true",
+ help="If this argument is provided will make a comparison(check -j/--json argument)")
+
+ parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False,
+ help="Print debug information")
+ #parse the command line arguments
+ args = parser.parse_args()
+
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig()
+ logger = logging.getLogger(__name__)
+
+ if not args.statusdump and not args.compare:
+ logger.error("At least -s/--statusdump or -c/--compare argument must be provided")
+ sys.exit(-1)
+
+ path_to_tasks = os.path.join(args.tasks, "tasks")
+ if not os.path.isdir(path_to_tasks):
+ logger.error("tasks directory could not be found in given path. aborting...")
+ sys.exit(-1)
+
+ logger.debug("Reading task files from directory {0}".format(path_to_tasks))
+ tasks = [ os.path.join(path_to_tasks, fold) for fold in os.listdir(path_to_tasks) if not fold.startswith('.') ]
+ giventasks = load_tasks(tasks)
+
+ if args.statusdump:
+ logger.debug("Status dump was selected")
+
+ with open(args.output, "w") as fout:
+ logger.debug("Dumping json dependencies file into {0}".format(args.output))
+ json.dump(giventasks, fout)
+
+ sys.exit(0)
+
+ if args.compare:
+ logger.debug("Comparing task files")
+ logger.debug("Reading json file: {0}".format(args.jsonfile))
+ with open(args.jsonfile, 'r') as json_input:
+ jsontasks = json.load(json_input)
+
+ logger.debug("Comparing tasks...")
+ compare_tasks(jsontasks, giventasks)
+
diff --git a/tasks_diff.py b/tasks_diff.py
deleted file mode 100755
index f241bdb..0000000
--- a/tasks_diff.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2013: Emmanouil Kiagias <e.kiagias at gmail.com>
-# License: GPL
-
-"""
-this module is still under construction, just testing stuff around
-before I write the complete script
-"""
-
-import re
-import pprint
-import logging
-from debian import deb822
-
-#code taken from udd/blends_metadata_gathener.py
-def clean_up_packages(packages):
- logger = logging.getLogger(__name__)
- # Hack: Debian Edu tasks files are using '\' at EOL which is broken
- # in RFC 822 files, but blend-gen-control from blends-dev relies
- # on this. So remove this stuff here for the Moment
- pkgs = re.sub('\\\\\n\s+', '', packages)
-
- # Remove versions from versioned depends
- pkgs = re.sub(' *\([ ><=\.0-9]+\) *', '', pkgs)
-
- # temporary strip spaces from alternatives ('|') to enable erroneous space handling as it was done before
- pkgs = re.sub('\s*\|\s*', '|', pkgs)
-
- # turn alternatives ('|') into real depends for this purpose
- # because we are finally interested in all alternatives
- pkgslist = pkgs.split(',')
- # Collect all dependencies in one line first,
- # create an object for each later
- pkgs_in_one_line = []
- for depl in pkgslist:
- dl = depl.strip()
- if dl != '': # avoid confusion when ',' is at end of line
- if re.search('\s', dl):
- logger.error("Blend %s task %s: Syntax error '%s'" % (blend, task, dl))
- # trying to fix the syntax error after issuing error message
- dlspaces = re.sub('\s+', ',', dl).split(',')
- for dls in dlspaces:
- pkgs_in_one_line.append(dls.strip())
- logger.info("Blend %s task %s: Found '%s' package inside broken syntax string - please fix task file anyway" % (blend, task, dls.strip()))
- else:
- # in case we have to deal with a set of alternatives
- if re.search('\|', dl):
- #for da in dl.split('|'):
- # deps_in_one_line.append(da)
- dl = re.sub('\|', ' | ', dl)
- pkgs_in_one_line.append(dl)
- # self.inject_package_alternatives(blend, task, strength, dl)
-
- return pkgs_in_one_line
-
-def load_task(path_to_task):
- """
- parses a task file and return a dictionary containing all its package headers elements
- (depends, suggests etc)
- """
- ftask = open(path_to_task, 'r')
-
- taskinfo = {}
- for header in ["depends", "suggests", "recommends", "avoid", "ignore"]:
- taskinfo[header] = []
-
- for paragraph in deb822.Sources.iter_paragraphs(ftask, shared_storage=False):
- if paragraph.has_key("task"):
- taskinfo["task"] = paragraph["task"]
-
- if paragraph.has_key("depends"):
- taskinfo["depends"] += clean_up_packages(paragraph["depends"])
-
- if paragraph.has_key("suggests"):
- taskinfo["suggests"] += clean_up_packages(paragraph["suggests"])
-
- if paragraph.has_key("recommends"):
- taskinfo["recommends"] += clean_up_packages(paragraph["recommends"])
-
- if paragraph.has_key("ignore"):
- taskinfo["ignore"] += clean_up_packages(paragraph["ignore"])
-
- if paragraph.has_key("avoid"):
- taskinfo["avoid"] += clean_up_packages(paragraph["avoid"])
-
- return taskinfo
-
-#in this function the input lists must be sorted
-def diff_sets(a_list, b_list):
- """
- return the elements which appear in a_list and not in b_list
- for example it can be used to compare two tasks from different releases
- and check which packages where removed/added.
- Note: in this function the input lists must be sorted
- """
- diffs = []
- b_index = 0
- b_list_len = len(b_list)
-
- for a_index, a in enumerate(a_list):
- for b in b_list[b_index:]:
- if a == b:
- b_index += 1
- break
- if a > b:
- b_index += 1
-
- if b_index + 1 == b_list_len:
- diffs += a_list[a_index:]
- break
-
- continue
- if a < b:
- diffs.append(a)
- break
-
- if b_index + 1 >= b_list_len:
- break
-
- return diffs
-
-
-#the input lists do not need to be sorted
-def diff_quicksort(a_list, b_list):
- """
- return the elements which appear in a_list and not in b_list
- for example it can be used to compare two tasks from different releases
- and check which packages where removed/added, a quicksort-similar algorithm is used
- """
- #value + reference
- a_array = [ { 'value' : a, 'ref' : False } for a in a_list ]
- b_array = [ { 'value' : b, 'ref' : True } for b in b_list ]
-
- return [ d['value'] for d in do_diff_quicksort(a_array+b_array) ]
-
-#same results as diff_sets
-#element_array: a+b and reference_array: b
-def do_diff_quicksort(element_array):
- array_len = len(element_array)
- if array_len <= 1:
- if element_array:
- if element_array[0]['ref']:
- return []
- return element_array
-
- #here we choose a pivot the middle element of the list
- pivot_index = int(array_len / 2)
- pivot = element_array[pivot_index]
-
- #remove pivot from list:
- del element_array[pivot_index]
-
- less = []
- greater = []
-
- include_pivot = True
- for x in element_array:
- if x['value'] < pivot['value']:
- less.append(x)
- elif x['value'] > pivot['value']:
- greater.append(x)
- else:
- include_pivot = False
-
- if include_pivot and not pivot['ref']:
- return do_diff_quicksort(less) + [pivot] + do_diff_quicksort(greater)
- else:
- return do_diff_quicksort(less) + do_diff_quicksort(greater)
-
-
-tasks_path = "trunk/debian-med/tasks"
-#yaw = load_task('/home/alamagestest/Projects/blends/projects/med/trunk/debian-med/tasks/bio')
-
-#these are already sorted
-spackages = ['im-switch', 'java-gcj-compat', 'kasteroids', 'katomic', 'kbabel', 'kbackgammon', 'kbattleship', 'lightspeed', 'mdns-scan', 'monopd', 'mozilla-mplayer']
-spackages2 = [ 'java-gcj-compat', 'kaffeine-mozilla', 'katomic', 'kbabel', 'kbackgammon', 'kbattleship', 'mdns-scan', 'monopd', 'mozilla-mplayer', 'mozilla-openoffice.org']
-
-#print diff_sets(packages, packages2)
-#print diff_sets(sorted(['x', 'y', 'z']), packages2)
-#pprint.pprint(yaw, indent=4)
-packages = [ 'monopd', 'mozilla-mplayer', 'im-switch','java-gcj-compat', 'kaffeine-mozilla', 'kasteroids', 'katomic', 'kbabel', 'kbackgammon', 'kbattleship', 'lightspeed', 'mozilla-openoffice.org']
-packages2 = [ 'mdns-scan', 'monopd', 'mozilla-mplayer', 'java-gcj-compat', 'kaffeine-mozilla', 'katomic', 'kbabel', 'kbackgammon', 'kbattleship', 'mozilla-openoffice.org']
-packages3 = ['x', 'y', 'z']
-print diff_quicksort( packages, packages2)
-
-print diff_sets(spackages, spackages2)
\ No newline at end of file
diff --git a/tester b/tester
index fee6e31..d4c1046 100755
--- a/tester
+++ b/tester
@@ -4,7 +4,7 @@
# with both different {sec-}blend-gen-control scripts for all existing Blends'
# and compares the generated files
-DEBUG=0
+DEBUG=1
dodiff() {
for file in $1/*
@@ -37,6 +37,7 @@ do
echo ""
#remove them so in next taskdesc/taskdesc-sec loop it won't diff them over again
- rm taskdesc/*
- rm taskdesc-sec/*
+ #remove them that way so readme file won't be deleted
+ ls taskdesc/* | grep -v "readme" | xargs rm
+ ls taskdesc-sec/* | grep -v "readme" | xargs rm
done
--
Git repository for blends-gsoc code
More information about the Blends-commit
mailing list