[Python-modules-commits] [python-s3transfer] 01/04: New upstream version 0.1.9
Takaki Taniguchi
takaki at moszumanska.debian.org
Thu Nov 10 10:55:47 UTC 2016
This is an automated email from the git hooks/post-receive script.
takaki pushed a commit to branch master
in repository python-s3transfer.
commit 3ab764b5422ca31254b4e5354afae45faaf2e6a3
Author: TANIGUCHI Takaki <takaki at asis.media-as.org>
Date: Thu Nov 10 19:54:11 2016 +0900
New upstream version 0.1.9
---
.changes/0.0.1.json | 7 +
.changes/0.1.0.json | 27 +++
.changes/0.1.1.json | 7 +
.changes/0.1.2.json | 7 +
.changes/0.1.3.json | 12 ++
.changes/0.1.4.json | 12 ++
.changes/0.1.5.json | 12 ++
.changes/0.1.6.json | 7 +
.changes/0.1.7.json | 7 +
.changes/0.1.8.json | 7 +
.coveragerc | 4 +
.gitignore | 42 +++++
.travis.yml | 11 ++
ACCEPTANCE_TESTS.rst | 183 ++++++++++++++++++
CHANGELOG.rst | 70 +++++++
NOTICE.txt | 2 +
PKG-INFO | 31 ----
requirements-dev.txt | 3 +
s3transfer.egg-info/PKG-INFO | 31 ----
s3transfer.egg-info/SOURCES.txt | 49 -----
s3transfer.egg-info/dependency_links.txt | 1 -
s3transfer.egg-info/requires.txt | 5 -
s3transfer.egg-info/top_level.txt | 1 -
s3transfer/__init__.py | 2 +-
s3transfer/exceptions.py | 4 +
s3transfer/futures.py | 17 +-
scripts/ci/install | 29 +++
scripts/ci/run-integ-tests | 20 ++
scripts/ci/run-tests | 20 ++
scripts/new-change | 209 +++++++++++++++++++++
scripts/performance/benchmark | 132 +++++++++++++
scripts/performance/benchmark-download | 163 ++++++++++++++++
scripts/performance/benchmark-upload | 140 ++++++++++++++
scripts/performance/download-file | 77 ++++++++
scripts/performance/summarize | 309 +++++++++++++++++++++++++++++++
scripts/performance/upload-file | 76 ++++++++
scripts/stress/timeout | 84 +++++++++
setup.cfg | 8 +-
tests/unit/test_futures.py | 28 ++-
tox.ini | 11 ++
40 files changed, 1737 insertions(+), 130 deletions(-)
diff --git a/.changes/0.0.1.json b/.changes/0.0.1.json
new file mode 100644
index 0000000..5bcd55c
--- /dev/null
+++ b/.changes/0.0.1.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "manager",
+ "description": "Add boto3 s3 transfer logic to package. (`issue 2 <https://github.com/boto/s3transfer/pull/2>`__)",
+ "type": "feature"
+ }
+]
diff --git a/.changes/0.1.0.json b/.changes/0.1.0.json
new file mode 100644
index 0000000..db2ec56
--- /dev/null
+++ b/.changes/0.1.0.json
@@ -0,0 +1,27 @@
+[
+ {
+ "category": "copy",
+ "description": "Add support for managed copies.",
+ "type": "feature"
+ },
+ {
+ "category": "download",
+ "description": "Add support for downloading to a filename, seekable file-like object, and nonseekable file-like object.",
+ "type": "feature"
+ },
+ {
+ "category": "general",
+ "description": "Add ``TransferManager`` class. All public functionality for ``s3transfer`` is exposed through this class.",
+ "type": "feature"
+ },
+ {
+ "category": "subscribers",
+ "description": "Add subscriber interface. Currently supports on_queued, on_progress, and on_done status changes.",
+ "type": "feature"
+ },
+ {
+ "category": "upload",
+ "description": "Add support for uploading a filename, seekable file-like object, and nonseekable file-like object.",
+ "type": "feature"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.1.json b/.changes/0.1.1.json
new file mode 100644
index 0000000..80b0da2
--- /dev/null
+++ b/.changes/0.1.1.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "deadlock",
+ "description": "Fix deadlock issue described here: https://bugs.python.org/issue20319 with using concurrent.futures.wait",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.2.json b/.changes/0.1.2.json
new file mode 100644
index 0000000..c9753cb
--- /dev/null
+++ b/.changes/0.1.2.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "download",
+ "description": "Patch memory leak related to unnecessarily holding onto futures for downloads.",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.3.json b/.changes/0.1.3.json
new file mode 100644
index 0000000..07a03f6
--- /dev/null
+++ b/.changes/0.1.3.json
@@ -0,0 +1,12 @@
+[
+ {
+ "category": "delete",
+ "description": "Add a ``.delete()`` method to the transfer manager.",
+ "type": "feature"
+ },
+ {
+ "category": "seekable upload",
+ "description": "Fix issue where seeked position of seekable file for a nonmultipart upload was not being taken into account.",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.4.json b/.changes/0.1.4.json
new file mode 100644
index 0000000..b9e5dce
--- /dev/null
+++ b/.changes/0.1.4.json
@@ -0,0 +1,12 @@
+[
+ {
+ "category": "chunksize",
+ "description": "Automatically adjust the chunksize if it doesn't meet S3s requirements.",
+ "type": "feature"
+ },
+ {
+ "category": "Download",
+ "description": "Add support for downloading to special UNIX file by name",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.5.json b/.changes/0.1.5.json
new file mode 100644
index 0000000..985c106
--- /dev/null
+++ b/.changes/0.1.5.json
@@ -0,0 +1,12 @@
+[
+ {
+ "category": "Cntrl-C",
+ "description": "Fix issue of hangs when Cntrl-C happens for many queued transfers",
+ "type": "bugfix"
+ },
+ {
+ "category": "cancel",
+ "description": "Expose messages for cancels",
+ "type": "feature"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.6.json b/.changes/0.1.6.json
new file mode 100644
index 0000000..db6d5d5
--- /dev/null
+++ b/.changes/0.1.6.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "download",
+ "description": "Fix issue where S3 Object was not downloaded to disk when empty",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.7.json b/.changes/0.1.7.json
new file mode 100644
index 0000000..ece0549
--- /dev/null
+++ b/.changes/0.1.7.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "TransferManager",
+ "description": "Fix memory leak when using same client to create multiple TransferManagers",
+ "type": "bugfix"
+ }
+]
\ No newline at end of file
diff --git a/.changes/0.1.8.json b/.changes/0.1.8.json
new file mode 100644
index 0000000..4a2bfb3
--- /dev/null
+++ b/.changes/0.1.8.json
@@ -0,0 +1,7 @@
+[
+ {
+ "category": "download",
+ "description": "Support downloading to FIFOs.",
+ "type": "feature"
+ }
+]
\ No newline at end of file
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..80e5f02
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+branch = True
+include =
+ s3transfer/*
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7e61a9a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,42 @@
+*.py[co]
+*.DS_Store
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+var
+sdist
+develop-eggs
+.installed.cfg
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+.cache
+
+#Translations
+*.mo
+
+#Mr Developer
+.mr.developer.cfg
+
+# Emacs backup files
+*~
+
+# Eclipse IDE
+/.project
+/.pydevproject
+
+# IDEA IDE
+.idea*
+src/
+
+# Completions Index
+completions.idx
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..afd045d
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,11 @@
+language: python
+python:
+ - "2.6"
+ - "2.7"
+ - "3.3"
+ - "3.4"
+ - "3.5"
+sudo: false
+install:
+ - python scripts/ci/install
+script: python scripts/ci/run-tests
diff --git a/ACCEPTANCE_TESTS.rst b/ACCEPTANCE_TESTS.rst
new file mode 100644
index 0000000..34b60ec
--- /dev/null
+++ b/ACCEPTANCE_TESTS.rst
@@ -0,0 +1,183 @@
+S3 Acceptance Tests
+===================
+
+List of all of the various scenarios that need to be handled in implementing
+a S3 transfer manager.
+
+Upload Tests
+------------
+
+General
+~~~~~~~
+* [x] Upload single nonmultipart file
+* [x] Upload single multipart file
+* [x] Upload multiple nonmultipart files
+* [x] Upload multiple multipart files
+* [x] Failed/cancelled multipart upload is aborted and leaves no orphaned parts especially for:
+
+ * [x] Failure of ``UploadPart``
+ * [x] Failure of ``CompleteMultipartUpload``
+ * [x] Failure unrelated to making an API call during upload such as read failure
+
+* [ ] Ctrl-C of any upload does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin``
+* [ ] Upload empty file
+* [ ] Upload nonseekable nonmultipart binary stream
+* [ ] Upload nonseekable multipart binary stream
+
+
+Region
+~~~~~~
+* [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart upload.
+
+
+Validation
+~~~~~~~~~~
+* [ ] Before upload, validate upload size of file is less than 5 TB.
+* [ ] Before upload, modify chunksize to an acceptable size when needed:
+
+ * [ ] Make chunksize 5 MB when the provided chunksize is less
+ * [ ] Make chunksize 5 GB when the provided chunksize is more
+ * [ ] Increase chunksize till the maximum number of parts for multipart upload is less than or equal to 10,000 parts
+
+* [ ] Before upload, ensure upload is nonmultipart if the file size is less than 5 MB no matter the provided multipart threshold.
+
+
+Extra Parameters
+~~~~~~~~~~~~~~~~
+* [ ] Upload multipart and nonmultipart file with any of the following properties:
+
+ * [x] ACL's
+ * [x] CacheControl
+ * [x] ContentDisposition
+ * [x] ContentEncoding
+ * [x] ContentLanguage
+ * [x] ContentType
+ * [x] Expires
+ * [x] Metadata
+ * [x] Grants
+ * [x] StorageClass
+ * [x] SSE (including KMS)
+ * [ ] Website Redirect
+
+* [x] Upload multipart and nonmultipart file with a sse-c key
+* [x] Upload multipart and nonmultipart file with requester pays
+
+
+Performance
+~~~~~~~~~~~
+* [ ] Maximum memory usage does not grow linearly with linearly increasing file size for any upload.
+* [ ] Maximum memory usage does not grow linearly with linearly increasing number of uploads.
+
+
+Download Tests
+--------------
+
+General
+~~~~~~~
+* [x] Download single nonmultipart object
+* [x] Download single multipart object
+* [x] Download multiple nonmultipart objects
+* [x] Download multiple multipart objects
+* [x] Download of any object is written to temporary file and renamed to final filename once the object is completely downloaded
+* [x] Failed downloads of any object cleans up temporary file
+* [x] Provide a transfer size for any download in lieu of using HeadObject
+* [ ] Ctrl-C of any download does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin``
+* [ ] Download nonmultipart object as nonseekable binary stream
+* [ ] Download multipart object as nonseekable binary stream
+
+
+Region
+~~~~~~
+* [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart download.
+
+
+Retry Logic
+~~~~~~~~~~~
+* [x] Retry on connection related errors when downloading data
+* [ ] Compare MD5 to ``ETag`` and retry for mismatches if all following scenarios are met:
+
+ * If MD5 is available
+ * Response does not have a ``ServerSideEncryption`` header equal to ``aws:kms``
+ * Response does not have ``SSECustomerAlgorithm``
+ * ``ETag`` does not have ``-`` in its value indicating a multipart transfer
+
+
+Extra Parameters
+~~~~~~~~~~~~~~~~
+* [x] Download an object of a specific version
+* [x] Download an object encrypted with sse-c
+* [x] Download an object using requester pays
+
+
+Performance
+~~~~~~~~~~~
+* [ ] Maximum memory usage does not grow linearly with linearly increasing file size for any download.
+* [ ] Maximum memory usage does not grow linearly with linearly increasing number of downloads.
+
+
+Copy Tests
+----------
+
+General
+~~~~~~~
+* [x] Copy single nonmultipart object
+* [x] Copy single multipart object
+* [x] Copy multiple nonmultipart objects
+* [x] Copy multiple multipart objects
+* [x] Provide a transfer size for any copy in lieu of using HeadObject.
+* [x] Failed/cancelled multipart copy is aborted and leaves no orphaned parts
+* [ ] Ctrl-C of any copy does not hang and the wait time is ``avg(transfer_time_iter_chunk) * some_margin``
+
+
+Region
+~~~~~~
+* [ ] Provide no or incorrect region for sig4 and be able to redirect request in fewest amount of calls as possible for multipart copy.
+
+
+Validation
+~~~~~~~~~~
+* [ ] Before copy, modify chunksize to an acceptable size when needed:
+
+ * [ ] Make chunksize 5 MB when the provided chunksize is less
+ * [ ] Make chunksize 5 GB when the provided chunksize is more
+ * [ ] Increase chunksize till the maximum number of parts for multipart copy is less than or equal to 10,000 parts
+
+* [ ] Before copy, ensure copy is nonmultipart if the file size is less than 5 MB no matter the provided multipart threshold.
+
+
+Extra Parameters
+~~~~~~~~~~~~~~~~
+* [ ] Copy multipart and nonmultipart file with any of the following properties:
+
+ * [x] ACL's
+ * [x] CacheControl
+ * [x] ContentDisposition
+ * [x] ContentEncoding
+ * [x] ContentLanguage
+ * [x] ContentType
+ * [x] Expires
+ * [x] Metadata
+ * [x] Grants
+ * [x] StorageClass
+ * [x] SSE (including KMS)
+ * [ ] Website Redirect
+
+* [x] Copy multipart and nonmultipart copies with copy source parameters:
+
+ * [x] CopySourceIfMatch
+ * [x] CopySourceIfModifiedSince
+ * [x] CopySourceIfNoneMatch
+ * [x] CopySourceIfUnmodifiedSince
+
+* [x] Copy nonmultipart object with metadata directive and do not use metadata directive for multipart object
+* [x] Copy multipart and nonmultipart objects of a specific version
+* [x] Copy multipart and nonmultipart objects using requester pays
+* [x] Copy multipart and nonmultipart objects using a sse-c key
+* [x] Copy multipart and nonmultipart objects using a copy source sse-c key
+* [x] Copy multipart and nonmultipart objects using a copy source sse-c key and sse-c key
+
+
+Cross-Bucket
+~~~~~~~~~~~~
+* [ ] Copy single nonmultipart object across sigv4 regions
+* [ ] Copy single multipart object across sigv4 regions
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..ad25ead
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,70 @@
+=========
+CHANGELOG
+=========
+
+0.1.8
+=====
+
+* feature:download: Support downloading to FIFOs.
+
+
+0.1.7
+=====
+
+* bugfix:TransferManager: Fix memory leak when using same client to create multiple TransferManagers
+
+
+0.1.6
+=====
+
+* bugfix:download: Fix issue where S3 Object was not downloaded to disk when empty
+
+
+0.1.5
+=====
+
+* bugfix:Cntrl-C: Fix issue of hangs when Cntrl-C happens for many queued transfers
+* feature:cancel: Expose messages for cancels
+
+
+0.1.4
+=====
+
+* feature:chunksize: Automatically adjust the chunksize if it doesn't meet S3s requirements.
+* bugfix:Download: Add support for downloading to special UNIX file by name
+
+
+0.1.3
+=====
+
+* feature:delete: Add a ``.delete()`` method to the transfer manager.
+* bugfix:seekable upload: Fix issue where seeked position of seekable file for a nonmultipart upload was not being taken into account.
+
+
+0.1.2
+=====
+
+* bugfix:download: Patch memory leak related to unnecessarily holding onto futures for downloads.
+
+
+0.1.1
+=====
+
+* bugfix:deadlock: Fix deadlock issue described here: https://bugs.python.org/issue20319 with using concurrent.futures.wait
+
+
+0.1.0
+=====
+
+* feature:copy: Add support for managed copies.
+* feature:download: Add support for downloading to a filename, seekable file-like object, and nonseekable file-like object.
+* feature:general: Add ``TransferManager`` class. All public functionality for ``s3transfer`` is exposed through this class.
+* feature:subscribers: Add subscriber interface. Currently supports on_queued, on_progress, and on_done status changes.
+* feature:upload: Add support for uploading a filename, seekable file-like object, and nonseekable file-like object.
+
+
+0.0.1
+=====
+
+* feature:manager: Add boto3 s3 transfer logic to package. (`issue 2 <https://github.com/boto/s3transfer/pull/2>`__)
+
diff --git a/NOTICE.txt b/NOTICE.txt
new file mode 100644
index 0000000..3e616fd
--- /dev/null
+++ b/NOTICE.txt
@@ -0,0 +1,2 @@
+s3transfer
+Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
diff --git a/PKG-INFO b/PKG-INFO
deleted file mode 100644
index 79cedc3..0000000
--- a/PKG-INFO
+++ /dev/null
@@ -1,31 +0,0 @@
-Metadata-Version: 1.1
-Name: s3transfer
-Version: 0.1.8
-Summary: An Amazon S3 Transfer Manager
-Home-page: https://github.com/boto/s3transfer
-Author: Amazon Web Services
-Author-email: kyknapp1 at gmail.com
-License: Apache License 2.0
-Description: =====================================================
- s3transfer - An Amazon S3 Transfer Manager for Python
- =====================================================
-
- .. warning::
-
- This project is currently a work in progress. Please do not rely on
- this functionality in production as the interfaces may change over time.
-
- S3transfer is a Python library for managing Amazon S3 transfers.
-
-Platform: UNKNOWN
-Classifier: Development Status :: 1 - Planning
-Classifier: Intended Audience :: Developers
-Classifier: Natural Language :: English
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000..3ef8584
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,3 @@
+-r requirements-test.txt
+psutil>=4.1.0,<5.0.0
+tabulate==0.7.5
diff --git a/s3transfer.egg-info/PKG-INFO b/s3transfer.egg-info/PKG-INFO
deleted file mode 100644
index 79cedc3..0000000
--- a/s3transfer.egg-info/PKG-INFO
+++ /dev/null
@@ -1,31 +0,0 @@
-Metadata-Version: 1.1
-Name: s3transfer
-Version: 0.1.8
-Summary: An Amazon S3 Transfer Manager
-Home-page: https://github.com/boto/s3transfer
-Author: Amazon Web Services
-Author-email: kyknapp1 at gmail.com
-License: Apache License 2.0
-Description: =====================================================
- s3transfer - An Amazon S3 Transfer Manager for Python
- =====================================================
-
- .. warning::
-
- This project is currently a work in progress. Please do not rely on
- this functionality in production as the interfaces may change over time.
-
- S3transfer is a Python library for managing Amazon S3 transfers.
-
-Platform: UNKNOWN
-Classifier: Development Status :: 1 - Planning
-Classifier: Intended Audience :: Developers
-Classifier: Natural Language :: English
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
diff --git a/s3transfer.egg-info/SOURCES.txt b/s3transfer.egg-info/SOURCES.txt
deleted file mode 100644
index 8651c39..0000000
--- a/s3transfer.egg-info/SOURCES.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-LICENSE.txt
-MANIFEST.in
-README.rst
-requirements-test.txt
-setup.cfg
-setup.py
-s3transfer/__init__.py
-s3transfer/compat.py
-s3transfer/copies.py
-s3transfer/delete.py
-s3transfer/download.py
-s3transfer/exceptions.py
-s3transfer/futures.py
-s3transfer/manager.py
-s3transfer/subscribers.py
-s3transfer/tasks.py
-s3transfer/upload.py
-s3transfer/utils.py
-s3transfer.egg-info/PKG-INFO
-s3transfer.egg-info/SOURCES.txt
-s3transfer.egg-info/dependency_links.txt
-s3transfer.egg-info/requires.txt
-s3transfer.egg-info/top_level.txt
-tests/__init__.py
-tests/functional/__init__.py
-tests/functional/test_copy.py
-tests/functional/test_delete.py
-tests/functional/test_download.py
-tests/functional/test_manager.py
-tests/functional/test_upload.py
-tests/functional/test_utils.py
-tests/integration/__init__.py
-tests/integration/test_copy.py
-tests/integration/test_delete.py
-tests/integration/test_download.py
-tests/integration/test_s3transfer.py
-tests/integration/test_upload.py
-tests/unit/__init__.py
-tests/unit/test_compat.py
-tests/unit/test_copies.py
-tests/unit/test_delete.py
-tests/unit/test_download.py
-tests/unit/test_futures.py
-tests/unit/test_manager.py
-tests/unit/test_s3transfer.py
-tests/unit/test_subscribers.py
-tests/unit/test_tasks.py
-tests/unit/test_upload.py
-tests/unit/test_utils.py
\ No newline at end of file
diff --git a/s3transfer.egg-info/dependency_links.txt b/s3transfer.egg-info/dependency_links.txt
deleted file mode 100644
index 8b13789..0000000
--- a/s3transfer.egg-info/dependency_links.txt
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/s3transfer.egg-info/requires.txt b/s3transfer.egg-info/requires.txt
deleted file mode 100644
index f81d58e..0000000
--- a/s3transfer.egg-info/requires.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-botocore>=1.3.0,<2.0.0
-futures>=2.2.0,<4.0.0
-
-[:python_version=="2.6" or python_version=="2.7"]
-futures>=2.2.0,<4.0.0
\ No newline at end of file
diff --git a/s3transfer.egg-info/top_level.txt b/s3transfer.egg-info/top_level.txt
deleted file mode 100644
index 572c6a9..0000000
--- a/s3transfer.egg-info/top_level.txt
+++ /dev/null
@@ -1 +0,0 @@
-s3transfer
diff --git a/s3transfer/__init__.py b/s3transfer/__init__.py
index dcd3a0a..28fbceb 100644
--- a/s3transfer/__init__.py
+++ b/s3transfer/__init__.py
@@ -143,7 +143,7 @@ from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError
__author__ = 'Amazon Web Services'
-__version__ = '0.1.8'
+__version__ = '0.1.9'
class NullHandler(logging.Handler):
diff --git a/s3transfer/exceptions.py b/s3transfer/exceptions.py
index fca3573..7f3a138 100644
--- a/s3transfer/exceptions.py
+++ b/s3transfer/exceptions.py
@@ -27,6 +27,10 @@ class InvalidSubscriberMethodError(Exception):
pass
+class TransferNotDoneError(Exception):
+ pass
+
+
class FatalError(CancelledError):
"""A CancelledError raised from an error in the TransferManager"""
pass
diff --git a/s3transfer/futures.py b/s3transfer/futures.py
index 15e03d0..afecc40 100644
--- a/s3transfer/futures.py
+++ b/s3transfer/futures.py
@@ -17,7 +17,7 @@ import logging
import threading
from s3transfer.compat import MAXINT
-from s3transfer.exceptions import CancelledError
+from s3transfer.exceptions import CancelledError, TransferNotDoneError
from s3transfer.utils import FunctionContainer
from s3transfer.utils import TaskSemaphore
@@ -77,6 +77,14 @@ class TransferFuture(object):
"""Cancels the request associated with the TransferFuture"""
self._coordinator.cancel()
+ def set_exception(self, exception):
+ """Sets the exception on the future."""
+ if not self.done():
+ raise TransferNotDoneError(
+ 'set_exception can only be called once the transfer is '
+ 'complete.')
+ self._coordinator.set_exception(exception, override=True)
+
class TransferMeta(object):
"""Holds metadata about the TransferFuture"""
@@ -191,13 +199,16 @@ class TransferCoordinator(object):
self._result = result
self._status = 'success'
- def set_exception(self, exception):
+ def set_exception(self, exception, override=False):
"""Set an exception for the TransferFuture
Implies the TransferFuture failed.
+
+ :param exception: The exception that cause the transfer to fail.
+ :param override: If True, override any existing state.
"""
with self._lock:
- if not self.done():
+ if not self.done() or override:
self._exception = exception
self._status = 'failed'
diff --git a/scripts/ci/install b/scripts/ci/install
new file mode 100755
index 0000000..4515cc2
--- /dev/null
+++ b/scripts/ci/install
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+import os
+import sys
+from subprocess import check_call
+import shutil
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(REPO_ROOT)
+
+
+def run(command):
+ return check_call(command, shell=True)
+
+
+try:
+ # Has the form "major.minor"
+ python_version = os.environ['PYTHON_VERSION']
+except KeyError:
+ python_version = '.'.join([str(i) for i in sys.version_info[:2]])
+
+run('pip install -r requirements-test.txt')
+run('pip install coverage')
+if os.path.isdir('dist') and os.listdir('dist'):
+ shutil.rmtree('dist')
+run('python setup.py bdist_wheel')
+wheel_dist = os.listdir('dist')[0]
+run('pip install %s' % (os.path.join('dist', wheel_dist)))
diff --git a/scripts/ci/run-integ-tests b/scripts/ci/run-integ-tests
new file mode 100755
index 0000000..3118bde
--- /dev/null
+++ b/scripts/ci/run-integ-tests
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Don't run tests from the root repo dir.
+# We want to ensure we're importing from the installed
+# binary package not from the CWD.
+
+import os
+from subprocess import check_call
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(os.path.join(REPO_ROOT, 'tests'))
+
+
+def run(command):
+ return check_call(command, shell=True)
+
+
+run('nosetests --with-xunit --cover-erase --with-coverage '
+ '--cover-package s3transfer --cover-xml -v integration')
diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests
new file mode 100755
index 0000000..fd7e189
--- /dev/null
+++ b/scripts/ci/run-tests
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Don't run tests from the root repo dir.
+# We want to ensure we're importing from the installed
+# binary package not from the CWD.
+
+import os
+from subprocess import check_call
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(os.path.join(REPO_ROOT, 'tests'))
+
+
+def run(command):
+ return check_call(command, shell=True)
+
+
+run('nosetests --with-coverage --cover-erase --cover-package s3transfer '
+ '--with-xunit --cover-xml -v unit/ functional/')
diff --git a/scripts/new-change b/scripts/new-change
new file mode 100755
index 0000000..6d56220
--- /dev/null
+++ b/scripts/new-change
@@ -0,0 +1,209 @@
+#!/usr/bin/env python
+"""Generate a new changelog entry.
+
+Usage
+=====
+
+To generate a new changelog entry::
+
+ scripts/new-change
+
+This will open up a file in your editor (via the ``EDITOR`` env var).
+You'll see this template::
+
+ # Type should be one of: feature, bugfix
+ type:
+
+ # Category is the high level feature area.
+ # This can be a service identifier (e.g ``s3``),
+ # or something like: Paginator.
+ category:
+
+ # A brief description of the change. You can
+ # use github style references to issues such as
+ # "fixes #489", "boto/boto3#100", etc. These
+ # will get automatically replaced with the correct
+ # link.
+ description:
+
+Fill in the appropriate values, save and exit the editor.
+Make sure to commit these changes as part of your pull request.
+
+If, when your editor is open, you decide don't don't want to add a changelog
+entry, save an empty file and no entry will be generated.
+
+You can then use the ``scripts/gen-changelog`` to generate the
+CHANGELOG.rst file.
+
+"""
+import os
+import re
+import sys
+import json
+import string
+import random
+import tempfile
+import subprocess
+import argparse
+
+
+VALID_CHARS = set(string.ascii_letters + string.digits)
+CHANGES_DIR = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ '.changes'
+)
+TEMPLATE = """\
+# Type should be one of: feature, bugfix
+type: {change_type}
+
+# Category is the high level feature area.
+# This can be a service identifier (e.g ``s3``),
+# or something like: Paginator.
+category: {category}
+
+# A brief description of the change. You can
+# use github style references to issues such as
+# "fixes #489", "boto/boto3#100", etc. These
+# will get automatically replaced with the correct
+# link.
+description: {description}
+"""
+
+
+def new_changelog_entry(args):
+ # Changelog values come from one of two places.
+ # Either all values are provided on the command line,
+ # or we open a text editor and let the user provide
+ # enter their values.
+ if all_values_provided(args):
+ parsed_values = {
+ 'type': args.change_type,
+ 'category': args.category,
+ 'description': args.description,
+ }
+ else:
+ parsed_values = get_values_from_editor(args)
+ if has_empty_values(parsed_values):
+ sys.stderr.write(
+ "Empty changelog values received, skipping entry creation.\n")
+ return 1
+ replace_issue_references(parsed_values, args.repo)
+ write_new_change(parsed_values)
+ return 0
+
+
+def has_empty_values(parsed_values):
+ return not (parsed_values.get('type') and
+ parsed_values.get('category') and
... 1225 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-s3transfer.git
More information about the Python-modules-commits
mailing list