[med-svn] [Git][med-team/toil][master] d/watch: mark beta releases as such
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Wed Mar 19 09:43:13 GMT 2025
Michael R. Crusoe pushed to branch master at Debian Med / toil
Commits:
7a584a93 by Michael R. Crusoe at 2025-03-19T10:43:05+01:00
d/watch: mark beta releases as such
- - - - -
3 changed files:
- debian/changelog
- debian/watch
- − src/toil/lib/aws/utils.py.orig
Changes:
=====================================
debian/changelog
=====================================
@@ -1,3 +1,9 @@
+toil (8.0.0-2) UNRELEASED; urgency=medium
+
+ * d/watch: mark beta releases as such
+
+ -- Michael R. Crusoe <crusoe at debian.org> Wed, 19 Mar 2025 10:43:03 +0100
+
toil (8.0.0-1) unstable; urgency=medium
[ Michael R. Crusoe ]
=====================================
debian/watch
=====================================
@@ -1,5 +1,5 @@
# Compulsory line, this is a version 4 file
version=4
-opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*@ARCHIVE_EXT@)%@PACKAGE at -$1%" \
+opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*@ARCHIVE_EXT@)%@PACKAGE at -$1%,uversionmangle=s/b/~b/g" \
https://github.com/BD2KGenomics/toil/tags .*/releases/@ANY_VERSION at .tar.gz
=====================================
src/toil/lib/aws/utils.py.orig deleted
=====================================
@@ -1,504 +0,0 @@
-# Copyright (C) 2015-2021 Regents of the University of California
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import errno
-import logging
-import os
-import socket
-from typing import (Any,
- Callable,
- ContextManager,
- Dict,
- Iterable,
- Iterator,
- List,
- Optional,
- Set,
-<<<<<<< HEAD
- Tuple,
- cast)
-=======
- cast,
- TYPE_CHECKING)
->>>>>>> ce9c91c31 (Allow for not installing the mypy_boto3_* packages)
-from urllib.parse import ParseResult
-
-from toil.lib.aws import session, AWSRegionName, AWSServerErrors
-from toil.lib.misc import printq
-from toil.lib.retry import (DEFAULT_DELAYS,
- DEFAULT_TIMEOUT,
- get_error_code,
- get_error_status,
- old_retry,
- retry, ErrorCondition)
-
-if TYPE_CHECKING:
- from mypy_boto3_sdb.type_defs import AttributeTypeDef
- from mypy_boto3_s3.service_resource import Bucket, Object as S3Object
-
-try:
- from botocore.exceptions import ClientError, EndpointConnectionError
-except ImportError:
- ClientError = None # type: ignore
- EndpointConnectionError = None # type: ignore
- # AWS/boto extra is not installed
-
-logger = logging.getLogger(__name__)
-
-# These are error codes we expect from AWS if we are making requests too fast.
-# https://github.com/boto/botocore/blob/49f87350d54f55b687969ec8bf204df785975077/botocore/retries/standard.py#L316
-THROTTLED_ERROR_CODES = [
- 'Throttling',
- 'ThrottlingException',
- 'ThrottledException',
- 'RequestThrottledException',
- 'TooManyRequestsException',
- 'ProvisionedThroughputExceededException',
- 'TransactionInProgressException',
- 'RequestLimitExceeded',
- 'BandwidthLimitExceeded',
- 'LimitExceededException',
- 'RequestThrottled',
- 'SlowDown',
- 'PriorRequestNotComplete',
- 'EC2ThrottledException',
-]
-
- at retry(errors=[AWSServerErrors])
-def delete_iam_role(
- role_name: str, region: Optional[str] = None, quiet: bool = True
-) -> None:
- # TODO: the Boto3 type hints are a bit oversealous here; they want hundreds
- # of overloads of the client-getting methods to exist based on the literal
- # string passed in, to return exactly the right kind of client or resource.
- # So we end up having to wrap all the calls in casts, which kind of defeats
- # the point of a nice fluent method you can call with the name of the thing
- # you want; we should have been calling iam_client() and so on all along if
- # we wanted MyPy to be able to understand us. So at some point we should
- # consider revising our API here to be less annoying to explain to the type
- # checker.
- iam_client = session.client('iam', region_name=region)
- iam_resource = session.resource('iam', region_name=region)
- role = iam_resource.Role(role_name)
- # normal policies
- for attached_policy in role.attached_policies.all():
- printq(f'Now dissociating policy: {attached_policy.policy_name} from role {role.name}', quiet)
- role.detach_policy(PolicyArn=attached_policy.arn)
- # inline policies
- for inline_policy in role.policies.all():
- printq(f'Deleting inline policy: {inline_policy.policy_name} from role {role.name}', quiet)
- iam_client.delete_role_policy(RoleName=role.name, PolicyName=inline_policy.policy_name)
- iam_client.delete_role(RoleName=role_name)
- printq(f'Role {role_name} successfully deleted.', quiet)
-
-
- at retry(errors=[AWSServerErrors])
-def delete_iam_instance_profile(
- instance_profile_name: str, region: Optional[str] = None, quiet: bool = True
-) -> None:
- iam_resource = session.resource("iam", region_name=region)
- instance_profile = iam_resource.InstanceProfile(instance_profile_name)
- if instance_profile.roles is not None:
- for role in instance_profile.roles:
- printq(f'Now dissociating role: {role.name} from instance profile {instance_profile_name}', quiet)
- instance_profile.remove_role(RoleName=role.name)
- instance_profile.delete()
- printq(f'Instance profile "{instance_profile_name}" successfully deleted.', quiet)
-
-
- at retry(errors=[AWSServerErrors])
-def delete_sdb_domain(
- sdb_domain_name: str, region: Optional[str] = None, quiet: bool = True
-) -> None:
- sdb_client = session.client("sdb", region_name=region)
- sdb_client.delete_domain(DomainName=sdb_domain_name)
- printq(f'SBD Domain: "{sdb_domain_name}" successfully deleted.', quiet)
-
-
-def connection_reset(e: Exception) -> bool:
- """
- Return true if an error is a connection reset error.
- """
- # For some reason we get 'error: [Errno 104] Connection reset by peer' where the
- # English description suggests that errno is 54 (ECONNRESET) while the actual
- # errno is listed as 104. To be safe, we check for both:
- return isinstance(e, socket.error) and e.errno in (errno.ECONNRESET, 104)
-
-def connection_error(e: Exception) -> bool:
- """
- Return True if an error represents a failure to make a network connection.
- """
- return (connection_reset(e)
- or isinstance(e, EndpointConnectionError))
-
-
-# TODO: Replace with: @retry and ErrorCondition
-def retryable_s3_errors(e: Exception) -> bool:
- """
- Return true if this is an error from S3 that looks like we ought to retry our request.
- """
- return (connection_error(e)
- or (isinstance(e, ClientError) and get_error_status(e) in (429, 500))
- or (isinstance(e, ClientError) and get_error_code(e) in THROTTLED_ERROR_CODES)
- # boto3 errors
- or (isinstance(e, ClientError) and get_error_code(e) in THROTTLED_ERROR_CODES)
- or (isinstance(e, ClientError) and 'BucketNotEmpty' in str(e))
- or (isinstance(e, ClientError) and e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') == 409 and 'try again' in str(e))
- or (isinstance(e, ClientError) and e.response.get('ResponseMetadata', {}).get('HTTPStatusCode') in (404, 429, 500, 502, 503, 504)))
-
-
-def retry_s3(delays: Iterable[float] = DEFAULT_DELAYS, timeout: float = DEFAULT_TIMEOUT, predicate: Callable[[Exception], bool] = retryable_s3_errors) -> Iterator[ContextManager[None]]:
- """
- Retry iterator of context managers specifically for S3 operations.
- """
- return old_retry(delays=delays, timeout=timeout, predicate=predicate)
-
- at retry(errors=[AWSServerErrors])
-def delete_s3_bucket(
- s3_resource: "S3ServiceResource",
- bucket: str,
- quiet: bool = True
-) -> None:
- """
- Delete the given S3 bucket.
- """
- printq(f'Deleting s3 bucket: {bucket}', quiet)
-
- paginator = s3_resource.meta.client.get_paginator('list_object_versions')
- try:
- for response in paginator.paginate(Bucket=bucket):
- # Versions and delete markers can both go in here to be deleted.
- # They both have Key and VersionId, but there's no shared base type
- # defined for them in the stubs to express that. See
- # <https://github.com/vemel/mypy_boto3_builder/issues/123>. So we
- # have to do gymnastics to get them into the same list.
- to_delete: List[Dict[str, Any]] = cast(List[Dict[str, Any]], response.get('Versions', [])) + \
- cast(List[Dict[str, Any]], response.get('DeleteMarkers', []))
- for entry in to_delete:
- printq(f" Deleting {entry['Key']} version {entry['VersionId']}", quiet)
- s3_resource.meta.client.delete_object(Bucket=bucket, Key=entry['Key'], VersionId=entry['VersionId'])
- s3_resource.Bucket(bucket).delete()
- printq(f'\n * Deleted s3 bucket successfully: {bucket}\n\n', quiet)
- except s3_resource.meta.client.exceptions.NoSuchBucket:
- printq(f'\n * S3 bucket no longer exists: {bucket}\n\n', quiet)
-
-
-def create_s3_bucket(
- s3_resource: "S3ServiceResource",
- bucket_name: str,
- region: AWSRegionName,
-) -> "Bucket":
- """
- Create an AWS S3 bucket, using the given Boto3 S3 session, with the
- given name, in the given region.
-
- Supports the us-east-1 region, where bucket creation is special.
-
- *ALL* S3 bucket creation should use this function.
- """
- logger.debug("Creating bucket '%s' in region %s.", bucket_name, region)
- if region == "us-east-1": # see https://github.com/boto/boto3/issues/125
- bucket = s3_resource.create_bucket(Bucket=bucket_name)
- else:
- bucket = s3_resource.create_bucket(
- Bucket=bucket_name,
- CreateBucketConfiguration={"LocationConstraint": region},
- )
- return bucket
-
- at retry(errors=[ClientError])
-def enable_public_objects(bucket_name: str) -> None:
- """
- Enable a bucket to contain objects which are public.
-
- This adjusts the bucket's Public Access Block setting to not block all
- public access, and also adjusts the bucket's Object Ownership setting to a
- setting which enables object ACLs.
-
- Does *not* touch the *account*'s Public Access Block setting, which can
- also interfere here. That is probably best left to the account
- administrator.
-
- This configuration used to be the default, and is what most of Toil's code
- is written to expect, but it was changed so that new buckets default to the
- more restrictive setting
- <https://aws.amazon.com/about-aws/whats-new/2022/12/amazon-s3-automatically-enable-block-public-access-disable-access-control-lists-buckets-april-2023/>,
- with the expectation that people would write IAM policies for the buckets
- to allow public access if needed. Toil expects to be able to make arbitrary
- objects in arbitrary places public, and naming them all in an IAM policy
- would be a very awkward way to do it. So we restore the old behavior.
- """
-
- s3_client = session.client('s3')
-
- # Even though the new default is for public access to be prohibited, this
- # is implemented by adding new things attached to the bucket. If we remove
- # those things the bucket will default to the old defaults. See
- # <https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/>.
-
- # Stop blocking public access
- s3_client.delete_public_access_block(Bucket=bucket_name)
-
- # Stop using an ownership controls setting that prohibits ACLs.
- s3_client.delete_bucket_ownership_controls(Bucket=bucket_name)
-
-class NoBucketLocationError(Exception):
- """
- Error to represent that we could not get a location for a bucket.
- """
- pass
-
-def get_bucket_region(bucket_name: str, endpoint_url: Optional[str] = None, only_strategies: Optional[Set[int]] = None) -> str:
- """
- Get the AWS region name associated with the given S3 bucket, or raise NoBucketLocationError.
-
- Does not log at info level or above when this does not work; failures are expected in some contexts.
-
- Takes an optional S3 API URL override.
-
- :param only_strategies: For testing, use only strategies with 1-based numbers in this set.
- """
-
- s3_client = session.client('s3', endpoint_url=endpoint_url)
-
- def attempt_get_bucket_location() -> Optional[str]:
- """
- Try and get the bucket location from the normal API call.
- """
- return s3_client.get_bucket_location(Bucket=bucket_name).get('LocationConstraint', None)
-
- def attempt_get_bucket_location_from_us_east_1() -> Optional[str]:
- """
- Try and get the bucket location from the normal API call, but against us-east-1
- """
- # Sometimes we aren't allowed to GetBucketLocation. At least some of
- # the time, that's only true when we talk to whatever S3 API servers we
- # usually use, and we can get around this lack of permission by talking
- # to us-east-1 instead. We've been told that this is because us-east-1
- # is special and will answer the question when other regions won't.
- # See:
- # <https://ucsc-gi.slack.com/archives/C027D41M6UA/p1652819831740169?thread_ts=1652817377.594539&cid=C027D41M6UA>
- # It could also be because AWS open data buckets (which we tend to
- # encounter this problem for) tend to actually themselves be in
- # us-east-1.
- backup_s3_client = session.client('s3', region_name='us-east-1')
- return backup_s3_client.get_bucket_location(Bucket=bucket_name).get('LocationConstraint', None)
-
- def attempt_head_bucket() -> Optional[str]:
- """
- Try and get the bucket location from calling HeadBucket and inspecting
- the headers.
- """
- # If that also doesn't work, we can try HEAD-ing the bucket and looking
- # for an 'x-amz-bucket-region' header on the response, which can tell
- # us where the bucket is. See
- # <https://github.com/aws/aws-sdk-cpp/issues/844#issuecomment-383747871>
- info = s3_client.head_bucket(Bucket=bucket_name)
- return info['ResponseMetadata']['HTTPHeaders']['x-amz-bucket-region']
-
- # Compose a list of strategies we want to try in order, which may work.
- # None is an acceptable return type that actually means something.
- strategies: List[Callable[[], Optional[str]]] = []
- strategies.append(attempt_get_bucket_location)
- if not endpoint_url:
- # We should only try to talk to us-east-1 if we don't have a custom
- # URL.
- strategies.append(attempt_get_bucket_location_from_us_east_1)
- strategies.append(attempt_head_bucket)
-
- error_logs: List[Tuple[int, str]] = []
- for attempt in retry_s3():
- with attempt:
- for i, strategy in enumerate(strategies):
- if only_strategies is not None and i+1 not in only_strategies:
- # We want to test running without this strategy.
- continue
- try:
- location = bucket_location_to_region(strategy())
- logger.debug('Got bucket location from strategy %d', i + 1)
- return location
- except ClientError as e:
- if get_error_code(e) == 'AccessDenied' and not endpoint_url:
- logger.debug('Strategy %d to get bucket location did not work: %s', i + 1, e)
- error_logs.append((i + 1, str(e)))
- last_error: Exception = e
- # We were blocked with this strategy. Move on to the
- # next strategy which might work.
- continue
- else:
- raise
- except KeyError as e:
- # If we get a weird head response we will have a KeyError
- logger.debug('Strategy %d to get bucket location did not work: %s', i + 1, e)
- error_logs.append((i + 1, str(e)))
- last_error = e
-
- error_messages = []
- for rank, message in error_logs:
- error_messages.append(f"Strategy {rank} failed to get bucket location because: {message}")
- # If we get here we ran out of attempts.
- raise NoBucketLocationError("Could not get bucket location: " + "\n".join(error_messages)) from last_error
-
-def region_to_bucket_location(region: str) -> str:
- return '' if region == 'us-east-1' else region
-
-def bucket_location_to_region(location: Optional[str]) -> str:
- return "us-east-1" if location == "" or location is None else location
-
-def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "S3Object":
- """
- Extracts a key (object) from a given parsed s3:// URL.
-
- If existing is true and the object does not exist, raises FileNotFoundError.
-
- :param bool existing: If True, key is expected to exist. If False, key is expected not to
- exists and it will be created. If None, the key will be created if it doesn't exist.
- """
-
- key_name = url.path[1:]
- bucket_name = url.netloc
-
- # Decide if we need to override Boto's built-in URL here.
- endpoint_url: Optional[str] = None
- host = os.environ.get('TOIL_S3_HOST', None)
- port = os.environ.get('TOIL_S3_PORT', None)
- protocol = 'https'
- if os.environ.get('TOIL_S3_USE_SSL', True) == 'False':
- protocol = 'http'
- if host:
- endpoint_url = f'{protocol}://{host}' + f':{port}' if port else ''
-
- # TODO: OrdinaryCallingFormat equivalent in boto3?
- # if botoargs:
- # botoargs['calling_format'] = boto.s3.connection.OrdinaryCallingFormat()
-
- try:
- # Get the bucket's region to avoid a redirect per request
- region = get_bucket_region(bucket_name, endpoint_url=endpoint_url)
- s3 = session.resource('s3', region_name=region, endpoint_url=endpoint_url)
- except NoBucketLocationError as e:
- # Probably don't have permission.
- # TODO: check if it is that
- logger.debug("Couldn't get bucket location: %s", e)
- logger.debug("Fall back to not specifying location")
- s3 = session.resource('s3', endpoint_url=endpoint_url)
-
- obj = s3.Object(bucket_name, key_name)
- objExists = True
-
- try:
- obj.load()
- except ClientError as e:
- if get_error_status(e) == 404:
- objExists = False
- else:
- raise
- if existing is True and not objExists:
- raise FileNotFoundError(f"Key '{key_name}' does not exist in bucket '{bucket_name}'.")
- elif existing is False and objExists:
- raise RuntimeError(f"Key '{key_name}' exists in bucket '{bucket_name}'.")
-
- if not objExists:
- obj.put() # write an empty file
- return obj
-
-
- at retry(errors=[AWSServerErrors])
-def list_objects_for_url(url: ParseResult) -> List[str]:
- """
- Extracts a key (object) from a given parsed s3:// URL. The URL will be
- supplemented with a trailing slash if it is missing.
- """
- key_name = url.path[1:]
- bucket_name = url.netloc
-
- if key_name != '' and not key_name.endswith('/'):
- # Make sure to put the trailing slash on the key, or else we'll see
- # a prefix of just it.
- key_name = key_name + '/'
-
- # Decide if we need to override Boto's built-in URL here.
- # TODO: Deduplicate with get_object_for_url, or push down into session module
- endpoint_url: Optional[str] = None
- host = os.environ.get('TOIL_S3_HOST', None)
- port = os.environ.get('TOIL_S3_PORT', None)
- protocol = 'https'
- if os.environ.get('TOIL_S3_USE_SSL', True) == 'False':
- protocol = 'http'
- if host:
- endpoint_url = f'{protocol}://{host}' + f':{port}' if port else ''
-
- client = session.client('s3', endpoint_url=endpoint_url)
-
- listing = []
-
- paginator = client.get_paginator('list_objects_v2')
- result = paginator.paginate(Bucket=bucket_name, Prefix=key_name, Delimiter='/')
- for page in result:
- if 'CommonPrefixes' in page:
- for prefix_item in page['CommonPrefixes']:
- listing.append(prefix_item['Prefix'][len(key_name):])
- if 'Contents' in page:
- for content_item in page['Contents']:
- if content_item['Key'] == key_name:
- # Ignore folder name itself
- continue
- listing.append(content_item['Key'][len(key_name):])
-
- logger.debug('Found in %s items: %s', url, listing)
- return listing
-
-def flatten_tags(tags: Dict[str, str]) -> List[Dict[str, str]]:
- """
- Convert tags from a key to value dict into a list of 'Key': xxx, 'Value': xxx dicts.
- """
- return [{'Key': k, 'Value': v} for k, v in tags.items()]
-
-
-def boto3_pager(requestor_callable: Callable[..., Any], result_attribute_name: str,
- **kwargs: Any) -> Iterable[Any]:
- """
- Yield all the results from calling the given Boto 3 method with the
- given keyword arguments, paging through the results using the Marker or
- NextToken, and fetching out and looping over the list in the response
- with the given attribute name.
- """
-
- # Recover the Boto3 client, and the name of the operation
- client = requestor_callable.__self__ # type: ignore[attr-defined]
- op_name = requestor_callable.__name__
-
- # grab a Boto 3 built-in paginator. See
- # <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html>
- paginator = client.get_paginator(op_name)
-
- for page in paginator.paginate(**kwargs):
- # Invoke it and go through the pages, yielding from them
- yield from page.get(result_attribute_name, [])
-
-
-def get_item_from_attributes(attributes: List["AttributeTypeDef"], name: str) -> Any:
- """
- Given a list of attributes, find the attribute associated with the name and return its corresponding value.
-
- The `attribute_list` will be a list of TypedDict's (which boto3 SDB functions commonly return),
- where each TypedDict has a "Name" and "Value" key value pair.
- This function grabs the value out of the associated TypedDict.
-
- If the attribute with the name does not exist, the function will return None.
-
- :param attributes: list of attributes
- :param name: name of the attribute
- :return: value of the attribute
- """
- return next((attribute["Value"] for attribute in attributes if attribute["Name"] == name), None)
View it on GitLab: https://salsa.debian.org/med-team/toil/-/commit/7a584a938dce67b3023345f065206c5780418537
--
View it on GitLab: https://salsa.debian.org/med-team/toil/-/commit/7a584a938dce67b3023345f065206c5780418537
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20250319/f0bc7082/attachment-0001.htm>
More information about the debian-med-commit
mailing list