[Blends-commit] [SCM] blends-gsoc branch, master, updated. 92734f7cd9782fde9a832a3fb2bd3a8636ad1419
Emmanouil Kiagias
e.kiagias at gmail.com
Wed Jul 24 21:04:46 UTC 2013
The following commit has been merged in the master branch:
commit 92734f7cd9782fde9a832a3fb2bd3a8636ad1419
Author: Emmanouil Kiagias <e.kiagias at gmail.com>
Date: Wed Jul 24 23:03:59 2013 +0200
Added single/blend-gen-control which generates a single control file and imitates the existing blend-gen-control. Both new scripts under the single/ folder seem to work *almost* fine, after some basic testing there seems to be a difference between the sql results(different exluded architectures) when quering for blends_dependencies(outer joins with packages for each architecture) and when quering for getting the available alternatives(straight query from packages<-- this seems to be ok), needs some investigation to resolve the sql bug.
diff --git a/blend-gen-control b/single/blend-gen-control
similarity index 73%
copy from blend-gen-control
copy to single/blend-gen-control
index 2b45a24..143155a 100755
--- a/blend-gen-control
+++ b/single/blend-gen-control
@@ -82,7 +82,7 @@ class UDD_connector:
#just check if any of the rows retured is empty
return [ release[0] for release in self.cursor.fetchall() if release[0] ]
- def get_available_architectures(self):
+ def __get_available_architectures(self):
"""
It queries the UDD and returns a list with all the available Debian architectures
"""
@@ -236,7 +236,18 @@ class UDD_connector:
For example the following lists: package_list = [ 'k3b', 'brasero' , 'k3b-i18n']
and alternatives = [ 'k3b | brasero' , 'k3b-i18n | brasero' ], and return this ['k3b | brasero', 'k3b-i18n | brasero']
"""
- package_list = kwargs["package_list"]
+ package_list_archs = kwargs["package_list"]
+ package_list = {}
+ for pkg_temp in package_list_archs:
+ if ' ' in pkg_temp:
+ splitted = pkg_temp.split(' ')
+ p = splitted[0].strip()
+
+ archs = ' '.join(splitted[1:])
+ package_list[p] = ' ' + archs
+ else:
+ package_list[pkg_temp] = ''
+
alternatives = kwargs["alternatives"]
dependency = kwargs["dependency"]
wanted_dependencies = kwargs["wanted_dependencies"]
@@ -263,21 +274,109 @@ class UDD_connector:
if real_temp:
if len(real_temp) == len(alt_exists):
- resolved_list.append(' | '.join(alt_exists))
+ fixed_alts = []
+ for a in alt_exists:
+ fixed_alts.append(a + package_list[a])
+
+ resolved_list.append(' | '.join(fixed_alts))
else:
virtual_temp = list ( set(alt_exists) - set(real_temp) )
- resolved_list.append(' | '.join( real_temp + virtual_temp ))
+ fixed_alts = []
+ for a in real_temp + virtual_temp:
+ fixed_alts.append(a + package_list[a])
+
+ resolved_list.append(' | '.join( fixed_alts ))
else:
excluded += alt_exists
else:
- resolved_list.append(' | '.join(alt_exists))
+ fixed_alts = []
+ for a in alt_exists:
+ fixed_alts.append(a + package_list[a])
+
+ resolved_list.append(' | '.join(fixed_alts))
for package in package_list:
if not package in seen:
- resolved_list.append(package)
+ resolved_list.append(package + package_list[package])
return resolved_list, excluded
+ def __resolve_architectures(self, exist_in_architectures, architectures):
+ """
+ return [ !arch1 !arch2 !arch3 ] as a string for the architecture differences between the given architectures list
+ """
+
+ if len(exist_in_architectures) == 1 and exist_in_architectures[0] == "all":
+ return ''
+
+ missing_archs = set(architectures) - set(exist_in_architectures)
+
+ if missing_archs:
+ excluded = []
+ for missing in missing_archs:
+ excluded.append('!' + missing)
+
+ return ' [' + ' '.join(excluded) + ']'
+ else:
+ return ''
+
+ def __build_all_architectures_query(self, blend, release, architectures):
+ """
+ Builds up a query to check each blends_dependency for each available
+ Debian architecture
+ """
+
+ select_clause = """
+ SELECT b.task, b.package, b.dependency, b.distribution, b.component, b.provides
+ """
+
+ from_clause = """
+ FROM blends_dependencies b
+ """
+
+ where_clause = """
+ WHERE b.blend='{0}'
+ """.format(blend)
+
+
+ for arch in architectures:
+ select_clause += ", pkg_{0}.architecture".format(arch.replace('-',''))
+
+ from_clause += """
+ LEFT OUTER JOIN (
+ SELECT p.package, p.architecture
+ FROM packages p JOIN releases r ON p.release = r.release
+ WHERE r.role='{0}' and architecture='{1}') pkg_{1} ON b.package = pkg_{1}.package
+ """.format(release, arch.replace('-',''))
+
+ return select_clause + from_clause + where_clause
+
+ def __get_available_virtuals(virtual_packages, release, architectures):
+ formatted_architectures = [ "'{0}'".format(arch) for arch in architectures ]
+
+ available_virtual = {}
+
+ query = """
+ SELECT distinct p.provides, p.architecture FROM packages p JOIN releases r ON p.release = r.release
+ WHERE r.role='{0}' AND p.distribution='debian' AND component='main' AND p.architecture in ( {1} )
+ AND provides ~ ('((\s|,)'||'({2})'||'(\s+|,|$)|^'||'({2})'||'$)')
+ """.format(release, ','.join(architectures), '|'.join(virtual_packages))
+
+ self.__execute_query(query)
+ row = self.cursor.fetchone()
+ while not row is None:
+ #row[0] : p.provides column
+ myprovides = [ x.strip() for x in row[0].split(',') ]
+ myarch = row[1]
+ for p in myprovides:
+ if not p in available_virtual:
+ available_virtual[p] = []
+
+ available_virtual[p].append(myarch)
+
+ row = self.cursor.fetchone()
+
+ return available_virtual
def get_blend_dependecies(self, **kwargs):
"""
@@ -288,7 +387,6 @@ class UDD_connector:
blend = kwargs["blend"]
release = kwargs["release"]
- architecture = kwargs["architecture"]
nodepends = kwargs["nodepends"]
taskdescription = kwargs['taskdescription']
@@ -300,35 +398,30 @@ class UDD_connector:
missing = []
excluded = []
- query = """
- SELECT b.task, b.package, b.dependency, b.distribution, b.component, pkg.architecture, b.provides
- FROM blends_dependencies b LEFT OUTER JOIN (
- SELECT p.package, p.architecture
- FROM packages p JOIN releases r ON p.release = r.release
- WHERE r.role='{1}' and architecture in ('{2}', 'all' )) pkg ON b.package = pkg.package
- WHERE b.blend='{0}'
- ORDER BY b.task
- """.format(blend, release, architecture)
+ architectures = self.__get_available_architectures()
+
+ query = self.__build_all_architectures_query(blend, release, architectures + ["all"])
self.__execute_query(query)
wanted_dependencies = []
if nodepends or taskdescription:
#in this case we need both depends and recommends
- wanted_dependencies += ['d', 'r']
-
+ wanted_dependencies += ['d', 'r']
else:
#in this case we only need depends
wanted_dependencies.append('d')
- #indexes of row: task(0), package(1), dependency(2), distribution(3), component(4), architecture(5), provides(6)
+ #indexes of row: task(0), package(1), dependency(2), distribution(3), component(4), provides(5)
+ #rest of the indexes are architectures
row = self.cursor.fetchone()
while not row is None:
increase_packages = True
#task, package, dependency, distribution, component, arch, provides = (row[0], row[1], row[2], row[3], row[4], row[5], row[6])
- task, package, dependency, distribution, component, arch, provides = row
-
+ task, package, dependency, distribution, component, provides = row[:6]
+ exist_in_archs = [ x for x in row[6:] if x ]
+
if provides:
if dependency in wanted_dependencies:
if not task in virtual_packages:
@@ -346,14 +439,17 @@ class UDD_connector:
#follow the same rules as the depends packages
#dependency 'd'== depends and 'r' == recommends
if dependency == 'd' or dependency == 'r':
- if distribution == 'debian' and component == 'main' and not arch is None:
- blend_dependencies[task]["Recommends"].append(package)
+ if distribution == 'debian' and component == 'main':
+ if exist_in_archs:
+ archs_resolved = self.__resolve_architectures(exist_in_archs, architectures)
+ blend_dependencies[task]["Recommends"].append(package + archs_resolved)
else:
blend_dependencies[task]["Suggests"].append(package)
else:
if dependency == 'd':
- if distribution == 'debian' and component == 'main' and not arch is None:
- blend_dependencies[task]["Depends"].append(package)
+ if distribution == 'debian' and component == 'main':
+ if exist_in_archs:
+ blend_dependencies[task]["Depends"].append(package + archs_resolved)
else:
blend_dependencies[task]["Suggests"].append(package)
elif dependency == 'r':
@@ -371,7 +467,7 @@ class UDD_connector:
increase_packages = False
#not sure if i should add these packages into the missing
- if not distribution == 'debian' or not component == 'main' or arch is None:
+ if not distribution == 'debian' or not component == 'main' or not exist_in_archs:
missing.append(package)
else:
available.append(package)
@@ -388,24 +484,7 @@ class UDD_connector:
#now we should handle the virtual packages if any:
if single_virtual_packages:
- available_virtual = []
-
- query = """
- SELECT distinct p.provides FROM packages p JOIN releases r ON p.release = r.release
- WHERE r.role='{0}' AND p.distribution='debian' AND component='main' AND p.architecture='{1}'
- AND provides ~ ('((\s|,)'||'({2})'||'(\s+|,|$)|^'||'({2})'||'$)')
- """.format(release, architecture, '|'.join(single_virtual_packages))
-
- self.__execute_query(query)
- row = self.cursor.fetchone()
- while not row is None:
- #row[0] : p.provides column
- myprovides = row[0]
- if ',' in myprovides:
- available_virtual += [ v.strip() for v in myprovides.split(',') ]
- else:
- available_virtual.append(row[0])
- row = self.cursor.fetchone()
+ available_virtual = self.__get_available_virtuals(single_virtual_packages, release, architectures + ["all"])
for task in virtual_packages:
exist = []
@@ -420,10 +499,11 @@ class UDD_connector:
existsInAlternatives = True
if existsInAlternatives:
+ archs_resolved = self.__resolve_architectures(available_virtual[pkg], architectures)
if nodepends or taskdescription:
- blend_dependencies[task]["Recommends"].append(pkg)
+ blend_dependencies[task]["Recommends"].append(pkg+archs_resolved)
else:
- blend_dependencies[task]["Depends"].append(pkg)
+ blend_dependencies[task]["Depends"].append(pkg+archs_resolved)
else:
blend_dependencies[task]["Suggests"].append(pkg)
@@ -460,8 +540,7 @@ class UDD_connector:
blend_dependencies[task]["Suggests"] += movetosuggest
#return the depenencies with the corrensponding architecture
- return ({ "architecture" : architecture, "tasks" : blend_dependencies }, { "available" : available, "architecture" : architecture},
- {"missing" : missing, "architecture" : architecture }, {'excluded' : excluded, 'architecture' : architecture})
+ return (blend_dependencies,available, missing, excluded)
def gen_control(**kwargs):
@@ -477,11 +556,11 @@ def gen_control(**kwargs):
suppressempty = kwargs["suppressempty"]
nodepends = kwargs["nodepends"]
tasksprefix = kwargs["blend_info"]["tasksprefix"]
- architecture = kwargs["blend_dependencies"]["architecture"]
- blend_dependencies = kwargs["blend_dependencies"]["tasks"]
+ blend_dependencies = kwargs["blend_dependencies"]
+ architecture = "any"
#TODO this is used for testing for the moment, will be changed
- control_path = "control/control.{0}".format(architecture)
+ control_path = "control"
logger.debug("Opening file {0} to write".format(control_path))
with open(control_path,'w') as fout:
@@ -529,67 +608,6 @@ def gen_control(**kwargs):
fout.write("\n")
-def gen_task_desc(**kwargs):
- """
- This method generates the task description file for tasksel
- """
- logger = logging.getLogger(__name__)
-
- suppressempty = kwargs["suppressempty"]
- blend = kwargs["blend_info"]["blend"]
- tasksprefix = kwargs["blend_info"]["tasksprefix"]
- architecture = kwargs["blend_dependencies"]["architecture"]
- blend_dependencies = kwargs["blend_dependencies"]["tasks"]
-
-
- #TODO this is used for testing for the moment, will be changed
- task_desc_path = "taskdesc/{0}-tasks.desc.{1}".format(blend, architecture)
- logger.debug("Opening file {0} to write".format(task_desc_path))
- with open(task_desc_path,'w') as fout:
-
- for task in sorted(blend_dependencies.keys()):
-
- if blend_dependencies[task]['Leaf'] == 'false':
- continue
-
- if suppressempty and blend_dependencies[task]["haspackages"] == 0:
- if blend_dependencies[task]['test_always_lang']:
- logger.debug("Print empty task {0} because Test-always-lang is set\n".format(task))
- else:
- logger.debug("The metapackage {2} will not be created because {0} dependant are in the pool and suppressempty was set {1}\n".format(blend_dependencies[task]["haspackages"], suppressempty, task))
- continue
-
- fout.write("Task: {0}-{1}\n".format(tasksprefix, task))
- fout.write("Section: {0}\n".format(blend));
- fout.write("Description: {0}\n".format(blend_dependencies[task]["description"]))
- fout.write("{0}".format(blend_dependencies[task]["long_description"])) #Already contains a newline
- fout.write("Relevance: 10\n")
-
- if blend_dependencies[task]["Enhances"]:
- fout.write("Enhances: {0}\n".format(blend_dependencies[task]["Enhances"]))
-
- if blend_dependencies[task]["metapackage"]:
- #No use listing a metapackage as a key package, if no metapackage exist.
- fout.write("Key: \n");
- fout.write(" {0}-{1}\n".format(tasksprefix, task))
-
- fout.write("Packages: list\n ")
- for header in ["Depends", "Recommends"]:
- if not blend_dependencies[task][header]:
- continue
- #Tasksel doesn't allow boolean(eg OR : |) comparisons in dependencies such as package1 | package2.
- #so we include in the list the first of the alternative packages
- alternatives_resolved = []
- for pkg in blend_dependencies[task][header]:
- if '|' in pkg:
- alternatives_resolved.append(pkg.split('|')[0])
- else:
- alternatives_resolved.append(pkg)
-
- fout.write("{0}".format("\n ".join(sorted(alternatives_resolved))))
-
- fout.write("\n\n")
-
def main():
default_release = "testing"
@@ -605,11 +623,6 @@ def main():
help="suppress tasks without any recommended package")
parser.add_argument("-c", dest="gencontrol", action="store_true", default=False,
help="Create new debian/control file.")
- parser.add_argument("-t", dest="taskdesc", action="store_true", default=False,
- help="Print task descriptions and package list for task")
- parser.add_argument("-a", "--architecture", dest="architecture", type=str,
- help="Target architecture, eg: i386, armel, amd64. If no architecture is provided,\
- files will produced for all available Debian architectures")
parser.add_argument("-d", "--debug", dest="debug", action="store_true", default=False,
help="Print debug information")
@@ -624,8 +637,8 @@ def main():
#----------------------------end of argparse and setup logging--------------------------------------#
#at least a -c or -t argument must be provided
- if not args.taskdesc and not args.gencontrol:
- logger.error("Argument -c(generate control file) or -t(generate task description file) is required.")
+ if not args.gencontrol:
+ logger.error("Argument -c(generate control file) is required.")
sys.exit(-1)
hasconfig = False
@@ -640,18 +653,6 @@ def main():
logger.error("Invalid release: {0}, aborting..".format(args.release))
sys.exit(-1)
- available_architectures = myudd.get_available_architectures()
-
- #check if the user provided a single architecture , if not use all the available
- if args.architecture:
- if args.architecture in available_architectures:
- architectures = [ args.architecture ]
- else:
- logger.error("Invalid architecture: {0}, aborting..".format(args.architecture))
- sys.exit(-1)
- else:
- architectures = available_architectures
-
#check if a config file exists
if os.path.isfile(config_file):
hasconfig = True
@@ -667,25 +668,13 @@ def main():
if args.gencontrol:
#generate a control for each provided architecture
- for arch in architectures:
- #get all the blends dependencies etc
- blend_dependencies, available, missing, excluded = myudd.get_blend_dependecies(blend = blend_info["blend"], release = release,
- architecture = arch, tasksprefix = blend_info["tasksprefix"], nodepends = nodepends, taskdescription = False)
-
- gen_control(blend_info = blend_info, blend_dependencies = blend_dependencies,
- suppressempty = suppressempty, nodepends = nodepends, hasconfig = hasconfig)
-
- elif args.taskdesc:
- #generate a task description for each provided architecture
- for arch in architectures:
- #we reuse the same code as above here BUT we need the blend_dependencies here without nodepends so we make sure we call it
- #with nodepends = False no matter the command line argument, no need to descrease depends to recommends in any way for task description
- blend_dependencies, available, missing, excluded = myudd.get_blend_dependecies(blend = blend_info["blend"], release = release,
- architecture = arch, tasksprefix = blend_info["tasksprefix"], nodepends = False, taskdescription = True)
-
- gen_task_desc(blend_info = blend_info, blend_dependencies = blend_dependencies,
- suppressempty = suppressempty)
-
+
+ #get all the blends dependencies etc
+ blend_dependencies, available, missing, excluded = myudd.get_blend_dependecies(blend = blend_info["blend"], release = release,
+ tasksprefix = blend_info["tasksprefix"], nodepends = nodepends, taskdescription = False)
+
+ gen_control(blend_info = blend_info, blend_dependencies = blend_dependencies,
+ suppressempty = suppressempty, nodepends = nodepends, hasconfig = hasconfig)
return 0
diff --git a/single/sec-blend-gen-control b/single/sec-blend-gen-control
index 3d63255..654b712 100755
--- a/single/sec-blend-gen-control
+++ b/single/sec-blend-gen-control
@@ -238,6 +238,9 @@ class UDD_connector:
row = self.cursor.fetchone()
+ for key , values in available_packages.items():
+ print key, values
+
return available_packages
def __build_virtuals_query(self, virtual_packages, release, architectures):
@@ -261,7 +264,7 @@ class UDD_connector:
#package value in index 0 and architecture value in index 1
row = self.cursor.fetchone()
while not row is None:
- pkg_temp_list = row[0].split(',')
+ pkg_temp_list = [ p.strip() for p in row[0].split(',') ]
for p in pkg_temp_list:
if not p in available_provides:
@@ -342,7 +345,9 @@ class UDD_connector:
if '|' in package:
#no need to handle alternatives or virtual which are not Debian,main
#because they will go into Suggests if they are Depends/Recommends
- if dependency in wanted_dependencies and distribution == 'debian' and component == 'main':
+ ###TODO check this out again
+ #if dependency in wanted_dependencies and distribution == 'debian' and component == 'main':
+ if dependency in wanted_dependencies:
if contains_provides:
virtual_packages += [ myalt.strip() for myalt in package.split("|") ]
--
Git repository for blends-gsoc code
More information about the Blends-commit
mailing list