[Python-modules-commits] [python-aptly] 01/03: Import python-aptly_0.10.orig.tar.gz
Filip Pytloun
fpytloun-guest at moszumanska.debian.org
Wed Sep 6 11:43:04 UTC 2017
This is an automated email from the git hooks/post-receive script.
fpytloun-guest pushed a commit to branch master
in repository python-aptly.
commit b9f0f09a71f6d1715cc6021759837ec3b1fa0996
Author: Filip Pytloun <filip at pytloun.cz>
Date: Wed Sep 6 13:29:15 2017 +0200
Import python-aptly_0.10.orig.tar.gz
---
.pep8speaks.yml | 5 ++
aptly/publisher/__init__.py | 145 +++++++++++++++++++++++++++++++++++++++++++-
aptly/publisher/__main__.py | 26 +++++++-
setup.py | 2 +-
4 files changed, 173 insertions(+), 5 deletions(-)
diff --git a/.pep8speaks.yml b/.pep8speaks.yml
new file mode 100644
index 0000000..04e9648
--- /dev/null
+++ b/.pep8speaks.yml
@@ -0,0 +1,5 @@
+pycodestyle:
+ ignore:
+ - E501
+scanner:
+ diff_only: true
diff --git a/aptly/publisher/__init__.py b/aptly/publisher/__init__.py
index 582d0c6..b1f6e72 100644
--- a/aptly/publisher/__init__.py
+++ b/aptly/publisher/__init__.py
@@ -3,11 +3,17 @@
import time
import re
import logging
+import yaml
from aptly.exceptions import AptlyException, NoSuchPublish
lg = logging.getLogger(__name__)
+def load_publish(publish):
+ with open(publish, 'r') as publish_file:
+ return yaml.load(publish_file)
+
+
class PublishManager(object):
"""
Manage multiple publishes
@@ -32,6 +38,37 @@ class PublishManager(object):
for dist in distributions:
self.publish(dist).add(snapshot, component)
+ def restore_publish(self, components, restore_file, recreate):
+ publish_file = load_publish(restore_file)
+ publish_source = Publish(self.client, publish_file.get('publish'))
+ publish_source.restore_publish(publish_file,
+ components=components,
+ recreate=recreate)
+
+ def dump_publishes(self, publishes_to_save, dump_dir, prefix):
+
+ if len(dump_dir) > 1 and dump_dir[-1] == '/':
+ dump_dir = dump_dir[:-1]
+
+ save_list = []
+ save_all = True
+
+ if publishes_to_save and not ('all' in publishes_to_save):
+ save_all = False
+
+ publishes = self.client.do_get('/publish')
+ for publish in publishes:
+ name = '{}/{}'.format(publish['Prefix'], publish['Distribution'])
+ if save_all or name in publishes_to_save:
+ save_list.append(Publish(self.client, name, load=True))
+
+ if not save_all and len(save_list) != len(publishes_to_save):
+ raise Exception('Publish(es) required not found')
+
+ for publish in save_list:
+ save_path = ''.join([dump_dir, '/', prefix, publish.name.replace('/', '-')])
+ publish.save_publish(save_path)
+
def _publish_match(self, publish, names=False):
"""
Check if publish name matches list of names or regex patterns
@@ -186,6 +223,109 @@ class Publish(object):
return publish
raise NoSuchPublish("Publish %s does not exist" % self.name)
+ def _remove_snapshots(self, snapshots):
+ for snapshot in snapshots:
+ self.client.do_delete('/snapshots/%s' % snapshot)
+
+ def save_publish(self, save_path):
+ """
+ Serialize publish in YAML
+ """
+ name = self.name.replace('/', '-')
+ timestamp = time.strftime("%Y%m%d%H%M%S")
+
+ yaml_dict = {}
+ yaml_dict["publish"] = self.name
+ yaml_dict["name"] = timestamp
+ yaml_dict["components"] = []
+ for component, snapshots in self.components.items():
+ packages = self.get_packages(component)
+ package_dict = []
+ for package in packages:
+ (arch, name, version, ref) = self.parse_package_ref(package)
+ package_dict.append({'package': name, 'version': version, 'arch': arch, 'ref': ref})
+ snapshot = self._find_snapshot(snapshots[0])
+
+ yaml_dict["components"].append({'component': component, 'snapshot': snapshot['Name'],
+ 'description': snapshot['Description'], 'packages': package_dict})
+
+ lg.info("Saving publish %s in %s" % (name, save_path))
+ with open(save_path, 'w') as save_file:
+ yaml.dump(yaml_dict, save_file, default_flow_style=False)
+
+ def restore_publish(self, config, components, recreate=False):
+ """
+ Restore publish from config file
+ """
+ if "all" in components:
+ components = []
+
+ try:
+ self.load()
+ publish = True
+ except NoSuchPublish:
+ publish = False
+
+ new_publish_snapshots = []
+ to_publish = []
+ created_snapshots = []
+
+ for saved_component in config.get('components', []):
+ component_name = saved_component.get('component')
+
+ if not component_name:
+ raise Exception("Corrupted file")
+
+ if components and component_name not in components:
+ continue
+
+ saved_packages = []
+ if not saved_component.get('packages'):
+ raise Exception("Component %s is empty" % component_name)
+
+ for package in saved_component.get('packages'):
+ package_ref = '{} {} {} {}'.format(package.get('arch'), package.get('package'), package.get('version'), package.get('ref'))
+ saved_packages.append(package_ref)
+
+ to_publish.append(component_name)
+
+ snapshot_name = '{}-{}'.format("restored", saved_component.get('snapshot'))
+ lg.debug("Creating snapshot %s for component %s of packages: %s"
+ % (snapshot_name, component_name, saved_packages))
+
+ try:
+ self.client.do_post(
+ '/snapshots',
+ data={
+ 'Name': snapshot_name,
+ 'SourceSnapshots': [],
+ 'Description': saved_component.get('description'),
+ 'PackageRefs': saved_packages,
+ }
+ )
+ created_snapshots.append(snapshot_name)
+ except AptlyException as e:
+ if e.res.status_code == 404:
+ # delete all the previously created
+ # snapshots because the file is corrupted
+ self._remove_snapshots(created_snapshots)
+ raise Exception("Source snapshot or packages don't exist")
+
+ new_publish_snapshots.append({
+ 'Component': component_name,
+ 'Name': snapshot_name
+ })
+
+ if components:
+ self.publish_snapshots = [x for x in self.publish_snapshots if x['Component'] not in components and x['Component'] not in to_publish]
+ check_components = [x for x in new_publish_snapshots if x['Component'] in components]
+ if len(check_components) != len(components):
+ self._remove_snapshots(created_snapshots)
+ raise Exception("Not possible to find all the components required in the backup file")
+
+ self.publish_snapshots += new_publish_snapshots
+ self.do_publish(recreate=recreate, merge_snapshots=False)
+
def load(self):
"""
Load publish info from remote
@@ -371,8 +511,9 @@ class Publish(object):
def do_publish(self, recreate=False, no_recreate=False,
force_overwrite=False, publish_contents=False,
- architectures=None):
- self.merge_snapshots()
+ architectures=None, merge_snapshots=True):
+ if merge_snapshots:
+ self.merge_snapshots()
try:
publish = self._get_publish()
except NoSuchPublish:
diff --git a/aptly/publisher/__main__.py b/aptly/publisher/__main__.py
index ce6bb36..e0d0b3f 100644
--- a/aptly/publisher/__main__.py
+++ b/aptly/publisher/__main__.py
@@ -33,7 +33,7 @@ def main():
parser = argparse.ArgumentParser("aptly-publisher")
group_common = parser.add_argument_group("Common")
- parser.add_argument('action', help="Action to perform (publish, promote, cleanup)")
+ parser.add_argument('action', help="Action to perform (publish, promote, cleanup, restore, dump)")
group_common.add_argument('-v', '--verbose', action="store_true")
group_common.add_argument('-d', '--debug', action="store_true")
group_common.add_argument('--dry', '--dry-run', action="store_true")
@@ -43,6 +43,7 @@ def main():
group_common.add_argument('--no-recreate', action="store_true", help="Never recreate publish (even when we are adding new components where it's the only option)")
group_common.add_argument('--force-overwrite', action="store_true", help="Overwrite files in pool/ directory without notice")
group_common.add_argument('--publish-contents', action="store_true", default=False, help="Publish contents. It's slow so disabled by default to support large repositories.")
+ group_common.add_argument('--components', nargs='+', help="Space-separated list of components to promote or restore")
group_publish = parser.add_argument_group("Action 'publish'")
group_publish.add_argument('-c', '--config', default="/etc/aptly/publisher.yaml", help="Configuration YAML file")
@@ -52,10 +53,17 @@ def main():
group_promote = parser.add_argument_group("Action 'promote'")
group_promote.add_argument('--source', help="Source publish to take snapshots from")
group_promote.add_argument('--target', help="Target publish to update")
- group_promote.add_argument('--components', nargs='+', help="Space-separated list of components to promote")
group_promote.add_argument('--packages', nargs='+', help="Space-separated list of packages to promote")
group_promote.add_argument('--diff', action="store_true", help="Show differences between publishes (snapshots to be updated)")
+ group_restore = parser.add_argument_group("Action 'restore'")
+ group_restore.add_argument('-r', '--restore-file', help="File used to restore publish")
+
+ group_save = parser.add_argument_group("Action 'dump'")
+ group_save.add_argument('-s', '--save-dir', help="Path of where dump of publish will be done")
+ group_save.add_argument('-x', '--prefix', default="saved-", help="Prefix for dump files' names")
+ group_save.add_argument('-p', '--publish', nargs='+', help="Space-separated list of publishes to save")
+
args = parser.parse_args()
if args.verbose:
@@ -86,6 +94,12 @@ def main():
elif args.action == 'cleanup':
publishmgr.cleanup_snapshots()
sys.exit(0)
+ elif args.action == 'dump':
+ action_dump(publishmgr, args.save_dir, args.publish, args.prefix)
+ elif args.action == "restore":
+ action_restore(publishmgr, components=args.components,
+ recreate=args.recreate,
+ restore_file=args.restore_file)
def action_promote(client, source, target, components=None, recreate=False,
@@ -163,6 +177,14 @@ def action_promote(client, source, target, components=None, recreate=False,
publish_contents=publish_contents)
+def action_dump(publishmgr, path, publish_to_save, prefix):
+ publishmgr.dump_publishes(publish_to_save, path, prefix)
+
+
+def action_restore(publishmgr, components, restore_file, recreate):
+ publishmgr.restore_publish(components, restore_file, recreate)
+
+
def action_diff(source, target, components=[], packages=True):
diff, equal = source.compare(target, components=components)
if not diff:
diff --git a/setup.py b/setup.py
index a32f654..c99c1df 100755
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ long_desc = open('README.rst').read()
setup(
name="python-aptly",
- version="0.9.1",
+ version="0.10",
description="Aptly REST API client and tooling",
long_description=long_desc,
author="Filip Pytloun",
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-aptly.git
More information about the Python-modules-commits
mailing list