[Python-modules-commits] [python-aptly] 01/03: import python-aptly_0.7.7.orig.tar.gz
Ondřej Nový
onovy at moszumanska.debian.org
Mon Aug 8 12:09:39 UTC 2016
This is an automated email from the git hooks/post-receive script.
onovy pushed a commit to branch master
in repository python-aptly.
commit 04ac733648ea10678bfa4902c3d7d23af06b3622
Author: Ondřej Nový <onovy at debian.org>
Date: Mon Aug 8 14:07:00 2016 +0200
import python-aptly_0.7.7.orig.tar.gz
---
.gitignore | 5 +
LICENSE | 339 +++++++++++++++++++++++++++++++++
README.rst | 216 +++++++++++++++++++++
aptly/__init__.py | 0
aptly/client.py | 91 +++++++++
aptly/exceptions.py | 11 ++
aptly/publisher/__init__.py | 405 ++++++++++++++++++++++++++++++++++++++++
aptly/publisher/__main__.py | 245 ++++++++++++++++++++++++
doc/aptly-publisher.png | Bin 0 -> 53388 bytes
doc/examples/ubuntu_mirror.yaml | 54 ++++++
doc/man/aptly-publisher.1 | 75 ++++++++
doc/publisher_diff_example.png | Bin 0 -> 30944 bytes
setup.py | 35 ++++
13 files changed, 1476 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6cf9b64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+*.swp
+*.o
+*.pyc
+.DS_Store
+.ropeproject
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..e4f6985
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,216 @@
+============
+python-aptly
+============
+
+Aptly REST API client and useful tooling
+
+Publisher
+=========
+
+Publisher is tooling for easier maintenance of complex repository management
+workflows.
+
+This is how workflow can look like and what publisher can do for you:
+
+.. image:: ./doc/aptly-publisher.png
+ :align: center
+
+Features
+--------
+
+- create or update publish from latest snapshots
+
+ - it takes configuration in yaml format which defines what to publish and
+ how
+ - expected snapshot format is ``<name>-<timestamp>``
+
+- promote publish
+
+ - use source publish snapshots to create or update another publish (eg.
+ testing -> stable)
+
+- cleanup unused snapshots
+
+Create or update publish
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+First create configuration file where you define Aptly repositories, mirrors
+and target distributions for publishing.
+
+.. code-block:: yaml
+
+ mirror:
+ # Ubuntu upstream repository
+ trusty-main:
+ # Base for our main component
+ component: main
+ distributions:
+ - nightly/trusty
+ # Mirrored 3rd party repository
+ aptly:
+ # Merge into main component
+ component: main
+ distributions:
+ - nightly/trusty
+
+ repo:
+ # Some repository with custom software
+ cloudlab:
+ # Publish as component cloudlab
+ component: cloudlab
+ distributions:
+ # We want to publish our packages (that can't break anything for
+ # sure) immediately to both nightly and testing repositories
+ - nightly/trusty
+ - testing/trusty
+
+Configuration above will create two publishes from latest snapshots of
+defined repositories and mirrors:
+
+- ``nightly/trusty`` with component cloudlab and main
+
+ - creates snapshot ``_main-<timestamp>`` by merging snapshots
+ ``aptly-<timestamp>`` and ``trusty-main-<timestamp>``)
+
+- ``testing/trusty`` with component cloudlab, made of repository cloudlab
+
+It expects that snapshots are already created (by mirror syncing script or by
+CI when new package is built) so it does following:
+
+- find latest snapshot (by creation date) for each defined mirror and
+ repository
+
+ - snapshots are recognized by name (eg. ``cloudlab-<timestamp>``,
+ ``trusty-main-<timestamp>``)
+
+- create new snapshot by merging snapshots with same publish component
+
+ - eg. create ``_main-<timestamp>`` from latest ``trusty-main-<timestamp>``
+ and ``aptly-<timestamp>`` snapshots
+ - merged snapshots are prefixed by ``_`` to avoid collisions with other
+ snapshots
+ - first it checks if merged snapshots already exists and if so, it will skip
+ creation of duplicated snapshot. So it's tries to be fully idempotent.
+
+- create or update publish or publishes as defined in configuration
+
+It can be executed like this:
+
+::
+
+ aptly-publisher -c config.yaml -v --url http://localhost:8080 publish
+
+Promote publish
+~~~~~~~~~~~~~~~
+
+Let's assume you have following prefixes and workflow:
+
+- nightly
+
+ - created by `publish` action when there's new snapshot or synced mirror
+ - packages are always up to date
+
+- testing
+
+ - freezed repository for testing and stabilization
+
+- stable
+
+ - well tested package versions
+ - well controlled update process
+
+There can be more publishes under prefix, eg. ``nightly/trusty``,
+``nightly/vivid``
+
+Then you need to switch published snapshots from one publish to another one.
+
+::
+
+ aptly-publisher -v --url http://localhost:8080 \
+ --source nightly/trusty --target testing/trusty \
+ publish
+
+You can also specify list of components. When you have separate components for
+your packages (eg. cloudlab) and security (mirror of trusty security
+repository), you may need to release them faster.
+
+::
+
+ aptly-publisher -v --url http://localhost:8080 \
+ --source nightly/trusty --target testing/trusty \
+ --components cloudlab security -- publish
+
+Finally you are also able to promote selected packages, eg.
+
+::
+
+ aptly-publisher -v --url http://localhost:8080 \
+ --source nightly/trusty --target testing/trusty \
+ --packages python-aptly aptly -- publish
+
+Show differences between publishes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can see differences between publishes with following command:
+
+::
+
+ aptly-publisher -v --url http://localhost:8080 \
+ --source nightly/trusty --target testing/trusty \
+ publish --diff
+
+Example output can look like this:
+
+.. image:: ./doc/publisher_diff_example.png
+ :align: center
+
+Cleanup unused snapshots
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+When you are creating snapshots regularly, you need to delete old ones that
+are not used by any publish. It's wise to call such action every time when
+publish is updated (eg. nightly).
+
+::
+
+ aptly-publisher -v --url http://localhost:8080 cleanup
+
+Installation
+============
+
+You can install directly using from local checkout or from pip:
+
+::
+
+ python setup.py install
+ pip install python-aptly
+
+
+Or better build Debian package with eg.:
+
+::
+
+ dpkg-buildpackage -uc -us
+
+Read more
+=========
+
+For usage informations, see ``aptly-publisher --help`` or man page.
+
+::
+
+ man man/aptly-publisher.1
+
+Also see ``doc/examples`` directory.
+
+For examples of jenkins jobs, have a look at `tcpcloud/jenkins-jobs <https://github.com/tcpcloud/jenkins-jobs>`_ repository.
+
+Known issues
+============
+
+- determine source snapshots correctly
+ (`#271 <https://github.com/smira/aptly/issues/271>`_)
+- cleanup merged snapshots before cleaning up source ones
+
+ - before that it's needed to run cleanup action multiple times to get all
+ unused snapshots cleaned
diff --git a/aptly/__init__.py b/aptly/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/aptly/client.py b/aptly/client.py
new file mode 100644
index 0000000..71952ff
--- /dev/null
+++ b/aptly/client.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+import logging
+from aptly.exceptions import AptlyException
+
+lg = logging.getLogger(__name__)
+
+
+class Aptly(object):
+ def __init__(self, url, auth=None, timeout=300, dry=False):
+ self.url = '%s%s' % (url, '/api')
+ self.timeout = timeout
+ self.dry = dry
+
+ self.session = requests.Session()
+ if auth is not None:
+ self.session.auth = auth
+ self.session.headers.update({
+ 'Accept': 'application/json',
+ 'Content-type': 'application/json',
+ })
+ self.api_version = self.get_version()
+
+ def get_version(self):
+ return self.do_get('/version')["Version"]
+
+ def _process_result(self, res):
+ if res.status_code < 200 or res.status_code >= 300:
+ raise AptlyException(
+ res,
+ "Something went wrong: %s (%s)" % (res.reason, res.status_code)
+ )
+ try:
+ return res.json()
+ except ValueError:
+ return res.text
+
+ def do_get(self, uri, kwargs=None, timeout=None):
+ url = '%s%s' % (self.url, uri)
+ lg.debug("GET %s, args=%s" % (url, kwargs))
+ res = self.session.get(
+ url,
+ timeout=timeout or self.timeout,
+ params=kwargs,
+ )
+ return self._process_result(res)
+
+ def do_post(self, uri, data, timeout=None):
+ data_json = json.dumps(data)
+ url = '%s%s' % (self.url, uri)
+ lg.debug("POST %s, data=%s" % (url, data_json))
+
+ if self.dry:
+ return
+
+ res = self.session.post(
+ url,
+ timeout=timeout or self.timeout,
+ data=data_json,
+ )
+ return self._process_result(res)
+
+ def do_delete(self, uri, timeout=None):
+ url = '%s%s' % (self.url, uri)
+ lg.debug("DELETE %s" % url)
+
+ if self.dry:
+ return
+
+ res = self.session.delete(
+ url,
+ timeout=timeout or self.timeout,
+ )
+ return self._process_result(res)
+
+ def do_put(self, uri, data, timeout=None):
+ data_json = json.dumps(data)
+ url = '%s%s' % (self.url, uri)
+ lg.debug("PUT %s, data=%s" % (url, data_json))
+
+ if self.dry:
+ return
+
+ res = self.session.put(
+ url,
+ timeout=timeout or self.timeout,
+ data=data_json,
+ )
+ return self._process_result(res)
diff --git a/aptly/exceptions.py b/aptly/exceptions.py
new file mode 100644
index 0000000..87b3342
--- /dev/null
+++ b/aptly/exceptions.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+
+class AptlyException(Exception):
+ def __init__(self, res, msg):
+ Exception.__init__(self, msg)
+ self.res = res
+
+
+class NoSuchPublish(Exception):
+ pass
diff --git a/aptly/publisher/__init__.py b/aptly/publisher/__init__.py
new file mode 100644
index 0000000..dc3dd04
--- /dev/null
+++ b/aptly/publisher/__init__.py
@@ -0,0 +1,405 @@
+# -*- coding: utf-8 -*-
+
+import time
+import re
+import logging
+from aptly.exceptions import AptlyException, NoSuchPublish
+
+lg = logging.getLogger(__name__)
+
+
+class PublishManager(object):
+ """
+ Manage multiple publishes
+ """
+ def __init__(self, client):
+ self.client = client
+ self._publishes = {}
+ self.timestamp = int(time.time())
+
+ def publish(self, distribution):
+ """
+ Get or create publish
+ """
+ try:
+ return self._publishes[distribution]
+ except KeyError:
+ self._publishes[distribution] = Publish(self.client, distribution, timestamp=self.timestamp)
+ return self._publishes[distribution]
+
+ def add(self, snapshot, distributions, component='main'):
+ """ Add mirror or repo to publish """
+ for dist in distributions:
+ self.publish(dist).add(snapshot, component)
+
+ def _publish_match(self, publish, names=False):
+ """
+ Check if publish name matches list of names or regex patterns
+ """
+ if names:
+ for name in names:
+ if isinstance(name, re._pattern_type):
+ if re.match(name, publish.name):
+ return True
+ else:
+ if publish in [name, './%s' % name]:
+ return True
+ return False
+ else:
+ return True
+
+ def do_publish(self, *args, **kwargs):
+ try:
+ publish_names = kwargs.pop('names')
+ except KeyError:
+ publish_names = None
+
+ for publish in self._publishes.itervalues():
+ if self._publish_match(publish.name, publish_names):
+ publish.do_publish(*args, **kwargs)
+ else:
+ lg.info("Skipping publish %s not matching publish names" % publish.name)
+
+ def list_uniq(self, seq):
+ keys = {}
+ for e in seq:
+ keys[e] = 1
+ return keys.keys()
+
+ def cleanup_snapshots(self):
+ snapshots = self.client.do_get('/snapshots', {'sort': 'time'})
+ exclude = []
+
+ # Add currently published snapshots into exclude list
+ publishes = self.client.do_get('/publish')
+ for publish in publishes:
+ exclude.extend(
+ [x['Name'] for x in publish['Sources']]
+ )
+
+ # Add last snapshots into exclude list
+ # TODO: ignore snapshots that are source for merged snapshots
+ snapshot_latest = []
+ for snapshot in snapshots:
+ base_name = snapshot['Name'].split('-')[0]
+ if base_name not in snapshot_latest:
+ snapshot_latest.append(base_name)
+ if snapshot['Name'] not in exclude:
+ lg.debug("Not deleting latest snapshot %s" % snapshot['Name'])
+ exclude.append(snapshot['Name'])
+
+ exclude = self.list_uniq(exclude)
+
+ for snapshot in snapshots:
+ if snapshot['Name'] not in exclude:
+ lg.info("Deleting snapshot %s" % snapshot['Name'])
+ try:
+ self.client.do_delete('/snapshots/%s' % snapshot['Name'])
+ except AptlyException as e:
+ if e.res.status_code == 409:
+ lg.warning("Snapshot %s is being used, can't delete" % snapshot['Name'])
+ else:
+ raise
+
+
+class Publish(object):
+ """
+ Single publish object
+ """
+ def __init__(self, client, distribution, timestamp=None, recreate=False, load=False, merge_prefix='_'):
+ self.client = client
+ self.recreate = recreate
+
+ dist_split = distribution.split('/')
+ self.distribution = dist_split[-1]
+ if dist_split[0] != self.distribution:
+ self.prefix = dist_split[0]
+ else:
+ self.prefix = ''
+
+ self.name = '%s/%s' % (self.prefix or '.', self.distribution)
+
+ if not timestamp:
+ self.timestamp = int(time.time())
+ else:
+ self.timestamp = timestamp
+
+ self.merge_prefix = merge_prefix
+ self.components = {}
+ self.publish_snapshots = []
+
+ if load:
+ # Load information from remote immediately
+ self.load()
+
+ def __eq__(self, other):
+ if not isinstance(other, Publish):
+ return False
+
+ diff, equal = self.compare(other)
+ if not diff:
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def compare(self, other, components=[]):
+ """
+ Compare two publishes
+ It expects that other publish is same or older than this one
+
+ Return tuple (diff, equal) of dict {'component': ['snapshot']}
+ """
+ lg.debug("Comparing publish %s and %s" % (self.name, other.name))
+
+ diff, equal = ({}, {})
+
+ for component, snapshots in self.components.iteritems():
+ if component not in other.components:
+ continue
+
+ if component not in other.components.keys():
+ # Component is missing in other
+ diff[component] = snapshots
+ continue
+
+ equal_snapshots = list(set(snapshots).intersection(other.components[component]))
+ if equal_snapshots:
+ lg.debug("Equal snapshots for %s: %s" % (component, equal_snapshots))
+ equal[component] = equal_snapshots
+
+ diff_snapshots = list(set(snapshots).difference(other.components[component]))
+ if diff_snapshots:
+ lg.debug("Different snapshots for %s: %s" % (component, diff_snapshots))
+ diff[component] = diff_snapshots
+
+ return (diff, equal)
+
+ def _get_publish(self):
+ """
+ Find this publish on remote
+ """
+ publishes = self.client.do_get('/publish')
+ for publish in publishes:
+ if publish['Distribution'] == self.distribution and \
+ publish['Prefix'] == (self.prefix or '.'):
+ return publish
+ raise NoSuchPublish("Publish %s does not exist" % self.name)
+
+ def load(self):
+ """
+ Load publish info from remote
+ """
+ publish = self._get_publish()
+ for source in publish['Sources']:
+ component = source['Component']
+ snapshot = source['Name']
+ self.publish_snapshots.append({
+ 'Component': component,
+ 'Name': snapshot
+ })
+
+ snapshot_remote = self._find_snapshot(snapshot)
+ for source in self._get_source_snapshots(snapshot_remote, fallback_self=True):
+ self.add(source, component)
+
+ def get_packages(self, component=None, components=[], packages=None):
+ """
+ Return package refs for given components
+ """
+ if component:
+ components = [component]
+
+ package_refs = []
+ for snapshot in self.publish_snapshots:
+ if component and snapshot['Component'] not in components:
+ # We don't want packages for this component
+ continue
+
+ component_refs = self.client.do_get('/snapshots/%s/packages' % snapshot['Name'])
+ if packages:
+ # Filter package names
+ for ref in component_refs:
+ if self.parse_package_ref(ref)[1] in packages:
+ package_refs.append(ref)
+ else:
+ package_refs.extend(component_refs)
+
+ return package_refs
+
+ def parse_package_ref(self, ref):
+ """
+ Return tuple of architecture, package_name, version, id
+ """
+ if not ref:
+ return None
+ parsed = re.match('(.*)\ (.*)\ (.*)\ (.*)', ref)
+ return parsed.groups()
+
+ def add(self, snapshot, component='main'):
+ """
+ Add snapshot of component to publish
+ """
+ try:
+ self.components[component].append(snapshot)
+ except KeyError:
+ self.components[component] = [snapshot]
+
+ def _find_snapshot(self, name):
+ """
+ Find snapshot on remote by name or regular expression
+ """
+ remote_snapshots = self.client.do_get('/snapshots', {'sort': 'time'})
+ for remote in reversed(remote_snapshots):
+ if remote["Name"] == name or \
+ re.match(name, remote["Name"]):
+ return remote
+ return None
+
+ def _get_source_snapshots(self, snapshot, fallback_self=False):
+ """
+ Get list of source snapshot names of given snapshot
+
+ TODO: we have to decide by description at the moment
+ """
+ if not snapshot:
+ return []
+
+ source_snapshots = re.findall(r"'([\w\d-]+)'", snapshot['Description'])
+ if not source_snapshots and fallback_self:
+ source_snapshots = [snapshot['Name']]
+
+ source_snapshots.sort()
+ return source_snapshots
+
+ def merge_snapshots(self):
+ """
+ Create component snapshots by merging other snapshots of same component
+ """
... 568 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-aptly.git
More information about the Python-modules-commits
mailing list