From 9b4b6abb068f7dede976ff6c207a00b2684ed579 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 16 Jun 2023 14:57:16 -0400 Subject: [PATCH 01/30] Factor on boto3 s3/sqs client/resource creation for optional override with S3_URL/SQS_URL env vars. --- CHANGELOG.rst | 5 +++++ dcicutils/beanstalk_utils.py | 7 ++++--- dcicutils/boto_s3.py | 25 +++++++++++++++++++++++++ dcicutils/boto_sqs.py | 25 +++++++++++++++++++++++++ dcicutils/deployment_utils.py | 3 ++- dcicutils/env_base.py | 4 ++-- dcicutils/env_manager.py | 4 ++-- dcicutils/env_scripts.py | 7 +++---- dcicutils/env_utils.py | 4 ++-- dcicutils/ff_utils.py | 3 ++- dcicutils/function_cache_decorator.py | 2 +- dcicutils/glacier_utils.py | 4 ++-- dcicutils/s3_utils.py | 6 +++--- pyproject.toml | 2 +- 14 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 dcicutils/boto_s3.py create mode 100644 dcicutils/boto_sqs.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a0fb177a0..d6a486fdf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,11 @@ dcicutils Change Log ---------- +7.5.2 +===== +* Added ``boto_s3`` module to create boto3 S3 client/resource so we can centrally use the S3_URL + environment variable to globally set S3 usage for a locally running ersatz instance of S3 like localstack. + 7.5.1 ===== diff --git a/dcicutils/beanstalk_utils.py b/dcicutils/beanstalk_utils.py index ba2b02c69..dbbfc79ec 100755 --- a/dcicutils/beanstalk_utils.py +++ b/dcicutils/beanstalk_utils.py @@ -19,6 +19,7 @@ beanstalk_info, describe_beanstalk_environments, get_beanstalk_real_url, compute_ff_prd_env, compute_ff_stg_env, compute_cgap_prd_env, compute_cgap_stg_env, compute_prd_env_for_env, ) +from .boto_s3 import boto_s3_client, boto_s3_resource from .common import REGION from .env_utils import is_stg_or_prd_env, is_orchestrated from .misc_utils import PRINT, exported, obsolete, remove_suffix, prompt_for_input @@ -1056,7 +1057,7 @@ def create_s3_buckets(new): 'elasticbeanstalk-%s-wfoutput' % new, 'elasticbeanstalk-%s-system' % new, ] - s3 = boto3.client('s3', region_name=REGION) + s3 = boto_s3_client(region_name=REGION) for bucket in new_buckets: s3.create_bucket(Bucket=bucket) PRINT('=== CREATED NEW S3 BUCKETS ===' % new) @@ -1083,7 +1084,7 @@ def delete_s3_buckets(env_name): # note that tibanna logs are shared so are not so easy to delete ] - s3 = boto3.resource('s3') + s3 = boto_s3_resource() for bucket in buckets: PRINT("deleting content for " + bucket) try: @@ -1281,7 +1282,7 @@ def copy_s3_buckets(new, old): 'elasticbeanstalk-%s-files' % old, 'elasticbeanstalk-%s-wfoutput' % old, ] - s3 = boto3.client('s3', region_name=REGION) + s3 = boto_s3_client(region_name=REGION) for bucket in new_buckets: try: s3.create_bucket(Bucket=bucket) diff --git a/dcicutils/boto_s3.py b/dcicutils/boto_s3.py new file mode 100644 index 000000000..8c4ebfc78 --- /dev/null +++ b/dcicutils/boto_s3.py @@ -0,0 +1,25 @@ +# The primary/initial purpose of this was to be able to use the S3_URL environment variable to refer +# to a locally running ersatz version of S3 via an emulator like localstack (https://localstack.cloud). + +import boto3 +import os + + +def boto_s3_client(**kwargs): + """ + Creates and returns a boto3 s3 client object. If the S3_URL environment variable is set then it + will use that value as the endpoint_url for the boto3 s3 client, unless an explicit endpoint_url + was passed in (via kwargs, per boto3.client convention) in which case that value will be used. + """ + s3_url = kwargs.get("endpoint_url") or os.environ.get("S3_URL") + return boto3.client("s3", endpoint_url=s3_url, **kwargs) if s3_url else boto3.client("s3", **kwargs) + + +def boto_s3_resource(**kwargs): + """ + Creates and returns a boto3 s3 resource object. If the S3_URL environment variable is set then it + will use that value as the endpoint_url for the boto3 s3 resource, unless an explicit endpoint_url + was passed in (via kwargs, per boto3.resource convention) in which case that value will be used. + """ + s3_url = kwargs.get("endpoint_url") or os.environ.get("S3_URL") + return boto3.resource("s3", endpoint_url=s3_url, **kwargs) if s3_url else boto3.resource("s3", **kwargs) diff --git a/dcicutils/boto_sqs.py b/dcicutils/boto_sqs.py new file mode 100644 index 000000000..354ad280c --- /dev/null +++ b/dcicutils/boto_sqs.py @@ -0,0 +1,25 @@ +# The primary/initial purpose of this was to be able to use the SQS_URL environment variable to refer +# to a locally running ersatz version of SQS via an emulator like localstack (https://localstack.cloud). + +import boto3 +import os + + +def boto_sqs_client(**kwargs): + """ + Creates and returns a boto3 sqs client object. If the SQS_URL environment variable is set then it + will use that value as the endpoint_url for the boto3 sqs client, unless an explicit endpoint_url + was passed in (via kwargs, per boto3.client convention) in which case that value will be used. + """ + sqs_url = kwargs.get("endpoint_url") or os.environ.get("SQS_URL") + return boto3.client("sqs", endpoint_url=sqs_url, **kwargs) if sqs_url else boto3.client("sqs", **kwargs) + + +def boto_sqs_resource(**kwargs): + """ + Creates and returns a boto3 sqs resource object. If the SQS_URL environment variable is set then it + will use that value as the endpoint_url for the boto3 sqs resource, unless an explicit endpoint_url + was passed in (via kwargs, per boto3.resource convention) in which case that value will be used. + """ + sqs_url = kwargs.get("endpoint_url") or os.environ.get("SQS_URL") + return boto3.resource("sqs", endpoint_url=sqs_url, **kwargs) if sqs_url else boto3.resource("sqs", **kwargs) diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index b79235d83..cfc031d66 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -34,6 +34,7 @@ def main(): from git import Repo from .base import compute_ff_prd_env, compute_cgap_prd_env +from .boto_s3 import boto_s3_client from .common import LEGACY_GLOBAL_ENV_BUCKET, LEGACY_CGAP_GLOBAL_ENV_BUCKET, DEFAULT_ECOSYSTEM from .env_utils import ( get_standard_mirror_env, data_set_for_env, get_bucket_env, @@ -113,7 +114,7 @@ def upload_application_to_s3(cls, zip_location): :param zip_location: where to find zip file to upload :return: True in success, False otherwise """ - s3_client = boto3.client('s3') + s3_client = boto_s3_client() return s3_client.put_object( ACL='public-read', Body=open(zip_location, 'rb'), diff --git a/dcicutils/env_base.py b/dcicutils/env_base.py index 2099421bd..590cd107f 100644 --- a/dcicutils/env_base.py +++ b/dcicutils/env_base.py @@ -1,8 +1,8 @@ -import boto3 import contextlib import os from typing import Optional +from .boto_s3 import boto_s3_resource from .common import LEGACY_GLOBAL_ENV_BUCKET, S3BucketName from .exceptions import ( # CannotInferEnvFromManyGlobalEnvs, @@ -72,7 +72,7 @@ def _get_configs(cls, env_bucket, kind): or cls.global_env_bucket_name() # but failing that, for legacy system, just use legacy name or cls._legacy_global_env_bucket_for_testing()) - s3_resource = boto3.resource('s3') + s3_resource = boto_s3_resource() env_bucket_model = s3_resource.Bucket(env_bucket_name) key_names = [key_obj.key for key_obj in env_bucket_model.objects.all()] configs = cls.filter_config_names(key_names, kind=kind) diff --git a/dcicutils/env_manager.py b/dcicutils/env_manager.py index 59279982b..7c23531f4 100644 --- a/dcicutils/env_manager.py +++ b/dcicutils/env_manager.py @@ -1,10 +1,10 @@ -import boto3 import botocore.client import json import logging import urllib.request from typing import Optional +from .boto_s3 import boto_s3_client from .env_base import EnvBase from .env_utils import full_env_name from .exceptions import CannotInferEnvFromManyGlobalEnvs, MissingGlobalEnv @@ -67,7 +67,7 @@ def __init__(self, env_name: Optional[str] = None, object, you should pass that. """ - self.s3 = s3 or boto3.client('s3') + self.s3 = s3 or boto_s3_client() if env_name and env_description: raise ValueError("You may only specify an env_name or an env_description") if env_description: diff --git a/dcicutils/env_scripts.py b/dcicutils/env_scripts.py index d38a2ebd1..ca52dd0bf 100644 --- a/dcicutils/env_scripts.py +++ b/dcicutils/env_scripts.py @@ -1,8 +1,7 @@ import argparse -import boto3 import json import yaml - +from .boto_s3 import boto_s3_client from .lang_utils import disjoined_list from .misc_utils import PRINT, print_error_message @@ -11,7 +10,7 @@ def get_envs_buckets(client=None): - client = client or boto3.client('s3') + client = client or boto_s3_client() all_buckets = [bucket['Name'] for bucket in client.list_buckets()['Buckets']] envs_buckets = [bucket for bucket in all_buckets if bucket.endswith("-envs")] return envs_buckets @@ -43,7 +42,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): single_key = key # print(f"bucket={bucket} mode={mode} single_key={single_key}") - client = boto3.client('s3') + client = boto_s3_client() if bucket is None: envs_buckets = get_envs_buckets(client=client) diff --git a/dcicutils/env_utils.py b/dcicutils/env_utils.py index 843952079..cd1342a1b 100644 --- a/dcicutils/env_utils.py +++ b/dcicutils/env_utils.py @@ -1,4 +1,3 @@ -import boto3 import contextlib import copy import functools @@ -10,6 +9,7 @@ from typing import Optional from urllib.parse import urlparse from . import env_utils_legacy as legacy +from .boto_s3 import boto_s3_client from .common import ( EnvName, OrchestratedApp, APP_FOURFRONT, APP_CGAP, ChaliceStage, CHALICE_STAGE_DEV, CHALICE_STAGE_PROD, UrlString, ) @@ -455,7 +455,7 @@ def _get_config_object_from_s3(cls, env_bucket: str, config_key: str) -> dict: :param config_key: the name of a key (the name of a configuration - an ff_env or ecosystem) """ try: - s3 = boto3.client('s3') + s3 = boto_s3_client() metadata = s3.get_object(Bucket=env_bucket, Key=config_key) data = json.load(metadata['Body']) return data diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index e50ececf8..b0fb3a13f 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -11,6 +11,7 @@ from typing import Optional, Dict, List from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from . import s3_utils, es_utils +from .boto_sqs import boto_sqs_client from .common import ( # KeyValuestringDictList, KeyValuestringDict, AnyAuthDict, AuthDict, SimpleAuthPair, AuthData, AnyAuthData, PortalEnvName, @@ -1341,7 +1342,7 @@ def internal_compute_stuff_in_queues(ff_env_index_namespace, check_secondary) -> raise ValueError(f"Must provide a full fourfront environment name to this function" f" (such as 'fourfront-webdev'). You gave: {ff_env_index_namespace!r}") stuff_in_queue = False - client = boto3.client('sqs', region_name='us-east-1') + client = boto_sqs_client(region_name='us-east-1') queue_names = ['-indexer-queue'] if check_secondary: queue_names.append('-secondary-indexer-queue') diff --git a/dcicutils/function_cache_decorator.py b/dcicutils/function_cache_decorator.py index 3db6620e4..c48749772 100644 --- a/dcicutils/function_cache_decorator.py +++ b/dcicutils/function_cache_decorator.py @@ -15,7 +15,7 @@ def function_cache(*decorator_args, **decorator_kwargs): Exactly analogous to the functools.lru_cache decorator, but also allows specifying that if the return value of the function is None then no caching is to be done; use the nocache_none decorator kwarg set to True to do this. And more generally, use - the nocache decorator kwart set to anything to not cache any value which equals that. + the nocache decorator kwarg set to anything to not cache any value which equals that. Also like @lru_cache, this supports the maxsize decorator argument, as well as the cache_info and cache_clear functions on the decorated function. diff --git a/dcicutils/glacier_utils.py b/dcicutils/glacier_utils.py index bbcf77893..901e50540 100644 --- a/dcicutils/glacier_utils.py +++ b/dcicutils/glacier_utils.py @@ -1,7 +1,7 @@ -import boto3 from typing import Union, List, Tuple from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm +from .boto_s3 import boto_s3_client from .common import ( S3_GLACIER_CLASSES, S3StorageClass, MAX_MULTIPART_CHUNKS, MAX_STANDARD_COPY_SIZE, ENCODED_LIFECYCLE_TAG_KEY @@ -52,7 +52,7 @@ def __init__(self, env_name: str): * patch_file_lifecycle_status * delete_glaciered_object_versions """ - self.s3 = boto3.client('s3') + self.s3 = boto_s3_client() self.env_name = env_name self.key_manager = CGAPKeyManager() self.env_key = self.key_manager.get_keydict_for_env(env_name) diff --git a/dcicutils/s3_utils.py b/dcicutils/s3_utils.py index e34a3f2b7..3f999b5ef 100644 --- a/dcicutils/s3_utils.py +++ b/dcicutils/s3_utils.py @@ -1,4 +1,3 @@ -import boto3 import json import logging import mimetypes @@ -9,6 +8,7 @@ from typing_extensions import Literal from zipfile import ZipFile from .base import get_beanstalk_real_url +from .boto_s3 import boto_s3_client, boto_s3_resource from .common import ( EnvName, # LegacyAuthDict, AnyAuthDict, AuthDict, @@ -100,7 +100,7 @@ def __init__(self, outfile_bucket=None, sys_bucket=None, raw_file_bucket=None, self.url = '' self._health_json_url = None self._health_json = None - self.s3 = boto3.client('s3', region_name='us-east-1') + self.s3 = boto_s3_client(region_name='us-east-1') global_bucket = EnvManager.global_env_bucket_name() self.env_manager = None # In a legacy environment, this will continue to be None if sys_bucket is None: @@ -446,7 +446,7 @@ def delete_key(self, key, bucket: Optional[str] = None) -> None: @classmethod def size(cls, bucket: str) -> int: """Slowly and expensively (if there are a lot of them), count the number of items in a bucket.""" - sbuck = boto3.resource('s3').Bucket(bucket) # get only head of objects so we can count them + sbuck = boto_s3_resource().Bucket(bucket) # get only head of objects so we can count them # There is apparently no sbuck.objects.count(), so one has to enumerate them all. # Although one avenue we didn't try that is mentioned in StackOverflow is to go to the billing department # in the AWS console, whch knows how many objects there are. diff --git a/pyproject.toml b/pyproject.toml index e29da5aab..2b929136e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.5.1" +version = "7.5.1.1b1" # TODO: To become 7.5.2 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 17551d3719d85ed7b6631e24e25675f5e6e14054 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 23 Jun 2023 18:31:43 -0400 Subject: [PATCH 02/30] Version update --- CHANGELOG.rst | 4 +++- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d6a486fdf..d46d17c2a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,10 +6,12 @@ dcicutils Change Log ---------- -7.5.2 +7.6.0 ===== * Added ``boto_s3`` module to create boto3 S3 client/resource so we can centrally use the S3_URL environment variable to globally set S3 usage for a locally running ersatz instance of S3 like localstack. +* Added ``boto_sqs`` module to create boto3 SQS client/resource so we can centrally use the SQS_URL + environment variable to globally set SQS usage for a locally running ersatz instance of SQS like localstack. 7.5.1 diff --git a/pyproject.toml b/pyproject.toml index 2b929136e..972552fda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.5.1.1b1" # TODO: To become 7.5.2 +version = "7.5.1.2b1" # TODO: To become 7.6.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From d136776d3c9dc43be123c05dd867ecb43ab422e9 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 08:03:52 -0400 Subject: [PATCH 03/30] boto3 imports --- dcicutils/env_base.py | 2 +- dcicutils/env_manager.py | 2 +- dcicutils/env_utils.py | 2 +- dcicutils/glacier_utils.py | 2 +- dcicutils/s3_utils.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dcicutils/env_base.py b/dcicutils/env_base.py index 590cd107f..cf2f048ce 100644 --- a/dcicutils/env_base.py +++ b/dcicutils/env_base.py @@ -2,7 +2,7 @@ import os from typing import Optional -from .boto_s3 import boto_s3_resource +from .boto_s3 import boto3, boto_s3_resource from .common import LEGACY_GLOBAL_ENV_BUCKET, S3BucketName from .exceptions import ( # CannotInferEnvFromManyGlobalEnvs, diff --git a/dcicutils/env_manager.py b/dcicutils/env_manager.py index 7c23531f4..ac5a7f91d 100644 --- a/dcicutils/env_manager.py +++ b/dcicutils/env_manager.py @@ -4,7 +4,7 @@ import urllib.request from typing import Optional -from .boto_s3 import boto_s3_client +from .boto_s3 import boto3, boto_s3_client from .env_base import EnvBase from .env_utils import full_env_name from .exceptions import CannotInferEnvFromManyGlobalEnvs, MissingGlobalEnv diff --git a/dcicutils/env_utils.py b/dcicutils/env_utils.py index cd1342a1b..e75ec78db 100644 --- a/dcicutils/env_utils.py +++ b/dcicutils/env_utils.py @@ -9,7 +9,7 @@ from typing import Optional from urllib.parse import urlparse from . import env_utils_legacy as legacy -from .boto_s3 import boto_s3_client +from .boto_s3 import boto3, boto_s3_client from .common import ( EnvName, OrchestratedApp, APP_FOURFRONT, APP_CGAP, ChaliceStage, CHALICE_STAGE_DEV, CHALICE_STAGE_PROD, UrlString, ) diff --git a/dcicutils/glacier_utils.py b/dcicutils/glacier_utils.py index 901e50540..729968544 100644 --- a/dcicutils/glacier_utils.py +++ b/dcicutils/glacier_utils.py @@ -1,7 +1,7 @@ from typing import Union, List, Tuple from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm -from .boto_s3 import boto_s3_client +from .boto_s3 import boto3, boto_s3_client from .common import ( S3_GLACIER_CLASSES, S3StorageClass, MAX_MULTIPART_CHUNKS, MAX_STANDARD_COPY_SIZE, ENCODED_LIFECYCLE_TAG_KEY diff --git a/dcicutils/s3_utils.py b/dcicutils/s3_utils.py index 3f999b5ef..8d35ecd93 100644 --- a/dcicutils/s3_utils.py +++ b/dcicutils/s3_utils.py @@ -8,7 +8,7 @@ from typing_extensions import Literal from zipfile import ZipFile from .base import get_beanstalk_real_url -from .boto_s3 import boto_s3_client, boto_s3_resource +from .boto_s3 import boto3, boto_s3_client, boto_s3_resource from .common import ( EnvName, # LegacyAuthDict, AnyAuthDict, AuthDict, From 9649ea689d4662eda5c438d6a1f9f81a74e40854 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 15:41:54 -0400 Subject: [PATCH 04/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- dcicutils/beanstalk_utils.py | 7 ++- dcicutils/boto_monkey_patching.py | 58 +++++++++++++++++++++++++ dcicutils/boto_s3.py | 25 ----------- dcicutils/boto_sqs.py | 25 ----------- dcicutils/deployment_utils.py | 3 +- dcicutils/env_base.py | 4 +- dcicutils/env_manager.py | 4 +- dcicutils/env_scripts.py | 7 +-- dcicutils/env_utils.py | 4 +- dcicutils/ff_utils.py | 3 +- dcicutils/function_cache_decorator.py | 2 +- dcicutils/glacier_utils.py | 4 +- dcicutils/s3_utils.py | 6 +-- test/conftest.py | 8 ++++ test/test_boto_monkey_patching.py | 61 +++++++++++++++++++++++++++ 15 files changed, 148 insertions(+), 73 deletions(-) create mode 100644 dcicutils/boto_monkey_patching.py delete mode 100644 dcicutils/boto_s3.py delete mode 100644 dcicutils/boto_sqs.py create mode 100644 test/test_boto_monkey_patching.py diff --git a/dcicutils/beanstalk_utils.py b/dcicutils/beanstalk_utils.py index dbbfc79ec..ba2b02c69 100755 --- a/dcicutils/beanstalk_utils.py +++ b/dcicutils/beanstalk_utils.py @@ -19,7 +19,6 @@ beanstalk_info, describe_beanstalk_environments, get_beanstalk_real_url, compute_ff_prd_env, compute_ff_stg_env, compute_cgap_prd_env, compute_cgap_stg_env, compute_prd_env_for_env, ) -from .boto_s3 import boto_s3_client, boto_s3_resource from .common import REGION from .env_utils import is_stg_or_prd_env, is_orchestrated from .misc_utils import PRINT, exported, obsolete, remove_suffix, prompt_for_input @@ -1057,7 +1056,7 @@ def create_s3_buckets(new): 'elasticbeanstalk-%s-wfoutput' % new, 'elasticbeanstalk-%s-system' % new, ] - s3 = boto_s3_client(region_name=REGION) + s3 = boto3.client('s3', region_name=REGION) for bucket in new_buckets: s3.create_bucket(Bucket=bucket) PRINT('=== CREATED NEW S3 BUCKETS ===' % new) @@ -1084,7 +1083,7 @@ def delete_s3_buckets(env_name): # note that tibanna logs are shared so are not so easy to delete ] - s3 = boto_s3_resource() + s3 = boto3.resource('s3') for bucket in buckets: PRINT("deleting content for " + bucket) try: @@ -1282,7 +1281,7 @@ def copy_s3_buckets(new, old): 'elasticbeanstalk-%s-files' % old, 'elasticbeanstalk-%s-wfoutput' % old, ] - s3 = boto_s3_client(region_name=REGION) + s3 = boto3.client('s3', region_name=REGION) for bucket in new_buckets: try: s3.create_bucket(Bucket=bucket) diff --git a/dcicutils/boto_monkey_patching.py b/dcicutils/boto_monkey_patching.py new file mode 100644 index 000000000..32af607f2 --- /dev/null +++ b/dcicutils/boto_monkey_patching.py @@ -0,0 +1,58 @@ +# Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url. +# Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the +# localstack utility, which provides a way to run some AWS services locally, for testing purposes. +# Currently only supported for S3 and SQS. To use this set the environment variables S3_URL and/or +# SQS_URL to the localstack URL, for example, http://localhost:4566. +# Reference: https://localstack.cloud + +import boto3 +import os +from typing import Optional + + +_boto_client_original = boto3.client +_boto_resource_original = boto3.resource +_boto_service_overrides_supported = [ + {"service": "s3", "env": "S3_URL"}, + {"service": "sqs", "env": "SQS_URL"} +] +# This will entirely disable this feature; for troubleshooting only. +_boto_monkey_patching_disabled = False + +# For import only in test_boto_monkey_patching. +_boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported] + + +# For import only in test_boto_monkey_patching. +def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]: + for item in _boto_service_overrides_supported: + if item["service"] == service: + return item["env"] + return None + + +def _setup_monkey_patching_kwargs(*args, **kwargs) -> dict: + if not _boto_monkey_patching_disabled: + endpoint_url = kwargs.get("endpoint_url") + if not endpoint_url: + for service_override in _boto_service_overrides_supported: + if service_override["service"] in args: + endpoint_url = os.environ.get(service_override["env"]) + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + break + return kwargs + + +def _monkey_patched_boto_client(*args, **kwargs): + kwargs = _setup_monkey_patching_kwargs(*args, **kwargs) + return _boto_client_original(*args, **kwargs) + + +def _monkey_patched_boto_resource(*args, **kwargs): + kwargs = _setup_monkey_patching_kwargs(*args, **kwargs) + return _boto_resource_original(*args, **kwargs) + + +boto3.client = _monkey_patched_boto_client +boto3.resource = _monkey_patched_boto_resource diff --git a/dcicutils/boto_s3.py b/dcicutils/boto_s3.py deleted file mode 100644 index 8c4ebfc78..000000000 --- a/dcicutils/boto_s3.py +++ /dev/null @@ -1,25 +0,0 @@ -# The primary/initial purpose of this was to be able to use the S3_URL environment variable to refer -# to a locally running ersatz version of S3 via an emulator like localstack (https://localstack.cloud). - -import boto3 -import os - - -def boto_s3_client(**kwargs): - """ - Creates and returns a boto3 s3 client object. If the S3_URL environment variable is set then it - will use that value as the endpoint_url for the boto3 s3 client, unless an explicit endpoint_url - was passed in (via kwargs, per boto3.client convention) in which case that value will be used. - """ - s3_url = kwargs.get("endpoint_url") or os.environ.get("S3_URL") - return boto3.client("s3", endpoint_url=s3_url, **kwargs) if s3_url else boto3.client("s3", **kwargs) - - -def boto_s3_resource(**kwargs): - """ - Creates and returns a boto3 s3 resource object. If the S3_URL environment variable is set then it - will use that value as the endpoint_url for the boto3 s3 resource, unless an explicit endpoint_url - was passed in (via kwargs, per boto3.resource convention) in which case that value will be used. - """ - s3_url = kwargs.get("endpoint_url") or os.environ.get("S3_URL") - return boto3.resource("s3", endpoint_url=s3_url, **kwargs) if s3_url else boto3.resource("s3", **kwargs) diff --git a/dcicutils/boto_sqs.py b/dcicutils/boto_sqs.py deleted file mode 100644 index 354ad280c..000000000 --- a/dcicutils/boto_sqs.py +++ /dev/null @@ -1,25 +0,0 @@ -# The primary/initial purpose of this was to be able to use the SQS_URL environment variable to refer -# to a locally running ersatz version of SQS via an emulator like localstack (https://localstack.cloud). - -import boto3 -import os - - -def boto_sqs_client(**kwargs): - """ - Creates and returns a boto3 sqs client object. If the SQS_URL environment variable is set then it - will use that value as the endpoint_url for the boto3 sqs client, unless an explicit endpoint_url - was passed in (via kwargs, per boto3.client convention) in which case that value will be used. - """ - sqs_url = kwargs.get("endpoint_url") or os.environ.get("SQS_URL") - return boto3.client("sqs", endpoint_url=sqs_url, **kwargs) if sqs_url else boto3.client("sqs", **kwargs) - - -def boto_sqs_resource(**kwargs): - """ - Creates and returns a boto3 sqs resource object. If the SQS_URL environment variable is set then it - will use that value as the endpoint_url for the boto3 sqs resource, unless an explicit endpoint_url - was passed in (via kwargs, per boto3.resource convention) in which case that value will be used. - """ - sqs_url = kwargs.get("endpoint_url") or os.environ.get("SQS_URL") - return boto3.resource("sqs", endpoint_url=sqs_url, **kwargs) if sqs_url else boto3.resource("sqs", **kwargs) diff --git a/dcicutils/deployment_utils.py b/dcicutils/deployment_utils.py index 339211e3a..69716a990 100644 --- a/dcicutils/deployment_utils.py +++ b/dcicutils/deployment_utils.py @@ -34,7 +34,6 @@ def main(): from git import Repo from .base import compute_ff_prd_env, compute_cgap_prd_env -from .boto_s3 import boto_s3_client from .common import LEGACY_GLOBAL_ENV_BUCKET, LEGACY_CGAP_GLOBAL_ENV_BUCKET, DEFAULT_ECOSYSTEM from .env_utils import ( get_standard_mirror_env, data_set_for_env, get_bucket_env, @@ -114,7 +113,7 @@ def upload_application_to_s3(cls, zip_location): :param zip_location: where to find zip file to upload :return: True in success, False otherwise """ - s3_client = boto_s3_client() + s3_client = boto3.client('s3') return s3_client.put_object( ACL='public-read', Body=open(zip_location, 'rb'), diff --git a/dcicutils/env_base.py b/dcicutils/env_base.py index cf2f048ce..2099421bd 100644 --- a/dcicutils/env_base.py +++ b/dcicutils/env_base.py @@ -1,8 +1,8 @@ +import boto3 import contextlib import os from typing import Optional -from .boto_s3 import boto3, boto_s3_resource from .common import LEGACY_GLOBAL_ENV_BUCKET, S3BucketName from .exceptions import ( # CannotInferEnvFromManyGlobalEnvs, @@ -72,7 +72,7 @@ def _get_configs(cls, env_bucket, kind): or cls.global_env_bucket_name() # but failing that, for legacy system, just use legacy name or cls._legacy_global_env_bucket_for_testing()) - s3_resource = boto_s3_resource() + s3_resource = boto3.resource('s3') env_bucket_model = s3_resource.Bucket(env_bucket_name) key_names = [key_obj.key for key_obj in env_bucket_model.objects.all()] configs = cls.filter_config_names(key_names, kind=kind) diff --git a/dcicutils/env_manager.py b/dcicutils/env_manager.py index ac5a7f91d..59279982b 100644 --- a/dcicutils/env_manager.py +++ b/dcicutils/env_manager.py @@ -1,10 +1,10 @@ +import boto3 import botocore.client import json import logging import urllib.request from typing import Optional -from .boto_s3 import boto3, boto_s3_client from .env_base import EnvBase from .env_utils import full_env_name from .exceptions import CannotInferEnvFromManyGlobalEnvs, MissingGlobalEnv @@ -67,7 +67,7 @@ def __init__(self, env_name: Optional[str] = None, object, you should pass that. """ - self.s3 = s3 or boto_s3_client() + self.s3 = s3 or boto3.client('s3') if env_name and env_description: raise ValueError("You may only specify an env_name or an env_description") if env_description: diff --git a/dcicutils/env_scripts.py b/dcicutils/env_scripts.py index ca52dd0bf..d38a2ebd1 100644 --- a/dcicutils/env_scripts.py +++ b/dcicutils/env_scripts.py @@ -1,7 +1,8 @@ import argparse +import boto3 import json import yaml -from .boto_s3 import boto_s3_client + from .lang_utils import disjoined_list from .misc_utils import PRINT, print_error_message @@ -10,7 +11,7 @@ def get_envs_buckets(client=None): - client = client or boto_s3_client() + client = client or boto3.client('s3') all_buckets = [bucket['Name'] for bucket in client.list_buckets()['Buckets']] envs_buckets = [bucket for bucket in all_buckets if bucket.endswith("-envs")] return envs_buckets @@ -42,7 +43,7 @@ def show_global_env_bucket(bucket, mode='json', key=None): single_key = key # print(f"bucket={bucket} mode={mode} single_key={single_key}") - client = boto_s3_client() + client = boto3.client('s3') if bucket is None: envs_buckets = get_envs_buckets(client=client) diff --git a/dcicutils/env_utils.py b/dcicutils/env_utils.py index e75ec78db..843952079 100644 --- a/dcicutils/env_utils.py +++ b/dcicutils/env_utils.py @@ -1,3 +1,4 @@ +import boto3 import contextlib import copy import functools @@ -9,7 +10,6 @@ from typing import Optional from urllib.parse import urlparse from . import env_utils_legacy as legacy -from .boto_s3 import boto3, boto_s3_client from .common import ( EnvName, OrchestratedApp, APP_FOURFRONT, APP_CGAP, ChaliceStage, CHALICE_STAGE_DEV, CHALICE_STAGE_PROD, UrlString, ) @@ -455,7 +455,7 @@ def _get_config_object_from_s3(cls, env_bucket: str, config_key: str) -> dict: :param config_key: the name of a key (the name of a configuration - an ff_env or ecosystem) """ try: - s3 = boto_s3_client() + s3 = boto3.client('s3') metadata = s3.get_object(Bucket=env_bucket, Key=config_key) data = json.load(metadata['Body']) return data diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index b0fb3a13f..e50ececf8 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -11,7 +11,6 @@ from typing import Optional, Dict, List from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from . import s3_utils, es_utils -from .boto_sqs import boto_sqs_client from .common import ( # KeyValuestringDictList, KeyValuestringDict, AnyAuthDict, AuthDict, SimpleAuthPair, AuthData, AnyAuthData, PortalEnvName, @@ -1342,7 +1341,7 @@ def internal_compute_stuff_in_queues(ff_env_index_namespace, check_secondary) -> raise ValueError(f"Must provide a full fourfront environment name to this function" f" (such as 'fourfront-webdev'). You gave: {ff_env_index_namespace!r}") stuff_in_queue = False - client = boto_sqs_client(region_name='us-east-1') + client = boto3.client('sqs', region_name='us-east-1') queue_names = ['-indexer-queue'] if check_secondary: queue_names.append('-secondary-indexer-queue') diff --git a/dcicutils/function_cache_decorator.py b/dcicutils/function_cache_decorator.py index c48749772..3db6620e4 100644 --- a/dcicutils/function_cache_decorator.py +++ b/dcicutils/function_cache_decorator.py @@ -15,7 +15,7 @@ def function_cache(*decorator_args, **decorator_kwargs): Exactly analogous to the functools.lru_cache decorator, but also allows specifying that if the return value of the function is None then no caching is to be done; use the nocache_none decorator kwarg set to True to do this. And more generally, use - the nocache decorator kwarg set to anything to not cache any value which equals that. + the nocache decorator kwart set to anything to not cache any value which equals that. Also like @lru_cache, this supports the maxsize decorator argument, as well as the cache_info and cache_clear functions on the decorated function. diff --git a/dcicutils/glacier_utils.py b/dcicutils/glacier_utils.py index 729968544..bbcf77893 100644 --- a/dcicutils/glacier_utils.py +++ b/dcicutils/glacier_utils.py @@ -1,7 +1,7 @@ +import boto3 from typing import Union, List, Tuple from concurrent.futures import ThreadPoolExecutor from tqdm import tqdm -from .boto_s3 import boto3, boto_s3_client from .common import ( S3_GLACIER_CLASSES, S3StorageClass, MAX_MULTIPART_CHUNKS, MAX_STANDARD_COPY_SIZE, ENCODED_LIFECYCLE_TAG_KEY @@ -52,7 +52,7 @@ def __init__(self, env_name: str): * patch_file_lifecycle_status * delete_glaciered_object_versions """ - self.s3 = boto_s3_client() + self.s3 = boto3.client('s3') self.env_name = env_name self.key_manager = CGAPKeyManager() self.env_key = self.key_manager.get_keydict_for_env(env_name) diff --git a/dcicutils/s3_utils.py b/dcicutils/s3_utils.py index 8d35ecd93..e34a3f2b7 100644 --- a/dcicutils/s3_utils.py +++ b/dcicutils/s3_utils.py @@ -1,3 +1,4 @@ +import boto3 import json import logging import mimetypes @@ -8,7 +9,6 @@ from typing_extensions import Literal from zipfile import ZipFile from .base import get_beanstalk_real_url -from .boto_s3 import boto3, boto_s3_client, boto_s3_resource from .common import ( EnvName, # LegacyAuthDict, AnyAuthDict, AuthDict, @@ -100,7 +100,7 @@ def __init__(self, outfile_bucket=None, sys_bucket=None, raw_file_bucket=None, self.url = '' self._health_json_url = None self._health_json = None - self.s3 = boto_s3_client(region_name='us-east-1') + self.s3 = boto3.client('s3', region_name='us-east-1') global_bucket = EnvManager.global_env_bucket_name() self.env_manager = None # In a legacy environment, this will continue to be None if sys_bucket is None: @@ -446,7 +446,7 @@ def delete_key(self, key, bucket: Optional[str] = None) -> None: @classmethod def size(cls, bucket: str) -> int: """Slowly and expensively (if there are a lot of them), count the number of items in a bucket.""" - sbuck = boto_s3_resource().Bucket(bucket) # get only head of objects so we can count them + sbuck = boto3.resource('s3').Bucket(bucket) # get only head of objects so we can count them # There is apparently no sbuck.objects.count(), so one has to enumerate them all. # Although one avenue we didn't try that is mentioned in StackOverflow is to go to the billing department # in the AWS console, whch knows how many objects there are. diff --git a/test/conftest.py b/test/conftest.py index 3b7de844e..154f60241 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,11 @@ import os + +# Disable any AWS endpoint-url environment variables overrides for testing. +if "S3_URL" in os.environ: + os.environ.pop("S3_URL") +if "SQS_URL" in os.environ: + os.environ.pop("SQS_URL") + import pytest import requests @@ -11,6 +18,7 @@ IntegratedFixture.verify_portal_access() + def _portal_health_get(namespace, portal_url, key): healh_json_url = f"{portal_url}/health?format=json" response = requests.get(healh_json_url) diff --git a/test/test_boto_monkey_patching.py b/test/test_boto_monkey_patching.py new file mode 100644 index 000000000..62c6bdbf1 --- /dev/null +++ b/test/test_boto_monkey_patching.py @@ -0,0 +1,61 @@ +import boto3 +import os +from dcicutils.misc_utils import override_environ + +# Reach into the implementation details of the boto_monkey_patching module +# to get list of boto3 services for which we support monkey patching, and to +# get the environment variable name used to do the associated endpoint-url override. +from dcicutils.boto_monkey_patching import _boto_monkey_patching_services +from dcicutils.boto_monkey_patching import _boto_monkey_patching_endpoint_url_environ_name + + +_override_endpoint_url = "http://localhost:4566" + + +def _is_default_aws_endpoint_url(endpoint_url: str, service: str): + return (endpoint_url == f"https://{service}.amazonaws.com" or + endpoint_url == f"https://{service}.{os.environ.get('AWS_DEFAULT_REGION')}.amazonaws.com") + + +def _test_boto_monkey_patching_client_without_overriding(service: str): + with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): None}): + s3_client = boto3.client(service) + assert _is_default_aws_endpoint_url(s3_client.meta.endpoint_url, service) + + +def _test_boto_monkey_patching_resource_without_overriding(service: str): + with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): None}): + s3_resource = boto3.resource(service) + assert _is_default_aws_endpoint_url(s3_resource.meta.client._endpoint.host, service) + + +def _test_boto_monkey_patching_client_with_overriding(service: str): + with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): _override_endpoint_url}): + s3_client = boto3.client(service) + assert s3_client.meta.endpoint_url == _override_endpoint_url + + +def _test_boto_monkey_patching_resource_with_overriding(service: str): + with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): _override_endpoint_url}): + s3_resource = boto3.resource(service) + assert s3_resource.meta.client._endpoint.host == _override_endpoint_url + + +def test_boto_monkey_patching_client_without_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_client_without_overriding(service) + + +def test_boto_monkey_patching_resource_without_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_resource_without_overriding(service) + + +def test_boto_monkey_patching_client_with_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_client_with_overriding(service) + + +def test_boto_monkey_patching_resource_with_overriding(): + for service in _boto_monkey_patching_services: + _test_boto_monkey_patching_resource_with_overriding(service) From c33cabd7d0989f5e71b3da3643bfc6f2357d4699 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 15:45:33 -0400 Subject: [PATCH 05/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- CHANGELOG.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4cd5dec69..1e465a199 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,13 +8,9 @@ Change Log 7.6.0 ===== -* Added ``boto_s3`` module to create boto3 S3 client/resource so we can centrally use the S3_URL - environment variable to globally set S3 usage for a locally running ersatz instance of S3 like localstack. -* Added ``boto_sqs`` module to create boto3 SQS client/resource so we can centrally use the SQS_URL - environment variable to globally set SQS usage for a locally running ersatz instance of SQS like localstack. -* TODO: Maybe a better way to do the above, globally setting endpoint URLs (to localstack versions) - without having to change existing boto3 calls to create client/resource objects; ChatGPT has - this to say about this process: https://chat.openai.com/share/49417c2c-0b99-4040-b38c-5ada37e080f5 +* Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for + for S3 or SQS boto3 client/resource creation using the S3_URL or SQS_URL environment variables + to specify that these services should use a locally running ersatz instance of S3 or SQS via localstack. 7.5.2 From 436ddd33666eb51cfeb0d7fbf387d15db5fb4902 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 15:48:01 -0400 Subject: [PATCH 06/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1e396bf3c..1462fbe92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.6.0.1b1" # TODO: To become 7.6.0 +version = "7.6.0.1b2" # TODO: To become 7.6.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 0051a6ec02a26f45a90012aaa6df8570ba2f2228 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 15:59:29 -0400 Subject: [PATCH 07/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- dcicutils/__init__.py | 1 + docs/source/dcicutils.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/dcicutils/__init__.py b/dcicutils/__init__.py index e69de29bb..bd4a9dc5d 100644 --- a/dcicutils/__init__.py +++ b/dcicutils/__init__.py @@ -0,0 +1 @@ +from . import boto_monkey_patching diff --git a/docs/source/dcicutils.rst b/docs/source/dcicutils.rst index d60150b41..28ad96f04 100644 --- a/docs/source/dcicutils.rst +++ b/docs/source/dcicutils.rst @@ -23,6 +23,13 @@ beanstalk_utils :members: +boto_monkey_patching +^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: dcicutils.boto_monkey_patching + :members: + + codebuild_utils ^^^^^^^^^^^^^^^ From 2ebe02469a60bbfb2c03cee5394eccafc5417a81 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:01:34 -0400 Subject: [PATCH 08/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- dcicutils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcicutils/__init__.py b/dcicutils/__init__.py index bd4a9dc5d..1842081dc 100644 --- a/dcicutils/__init__.py +++ b/dcicutils/__init__.py @@ -1 +1 @@ -from . import boto_monkey_patching +from . import boto_monkey_patching # noqa From 138227262da0d26d6606a78060c1f37b55dce290 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:14:27 -0400 Subject: [PATCH 09/30] flake8 --- test/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 154f60241..b59d69aee 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -18,7 +18,6 @@ IntegratedFixture.verify_portal_access() - def _portal_health_get(namespace, portal_url, key): healh_json_url = f"{portal_url}/health?format=json" response = requests.get(healh_json_url) From 25fcce18701a41b1a39aa4d33188186635a0106e Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:31:20 -0400 Subject: [PATCH 10/30] Suppport for monkey patched version of localstack-ization of boto3 s3 and sqs. --- test/test_boto_monkey_patching.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/test_boto_monkey_patching.py b/test/test_boto_monkey_patching.py index 62c6bdbf1..8b43010a2 100644 --- a/test/test_boto_monkey_patching.py +++ b/test/test_boto_monkey_patching.py @@ -1,5 +1,6 @@ import boto3 import os +from typing import Optional from dcicutils.misc_utils import override_environ # Reach into the implementation details of the boto_monkey_patching module @@ -17,26 +18,33 @@ def _is_default_aws_endpoint_url(endpoint_url: str, service: str): endpoint_url == f"https://{service}.{os.environ.get('AWS_DEFAULT_REGION')}.amazonaws.com") +def _environ_overrides(service: str, endpoint_url: Optional[str]) -> dict: + return { + _boto_monkey_patching_endpoint_url_environ_name(service): endpoint_url, + "AWS_DEFAULT_REGION": "us-east-1" + } + + def _test_boto_monkey_patching_client_without_overriding(service: str): - with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): None}): + with override_environ(**_environ_overrides(service, None)): s3_client = boto3.client(service) assert _is_default_aws_endpoint_url(s3_client.meta.endpoint_url, service) def _test_boto_monkey_patching_resource_without_overriding(service: str): - with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): None}): + with override_environ(**_environ_overrides(service, None)): s3_resource = boto3.resource(service) assert _is_default_aws_endpoint_url(s3_resource.meta.client._endpoint.host, service) def _test_boto_monkey_patching_client_with_overriding(service: str): - with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): _override_endpoint_url}): + with override_environ(**_environ_overrides(service, _override_endpoint_url)): s3_client = boto3.client(service) assert s3_client.meta.endpoint_url == _override_endpoint_url def _test_boto_monkey_patching_resource_with_overriding(service: str): - with override_environ(**{_boto_monkey_patching_endpoint_url_environ_name(service): _override_endpoint_url}): + with override_environ(**_environ_overrides(service, _override_endpoint_url)): s3_resource = boto3.resource(service) assert s3_resource.meta.client._endpoint.host == _override_endpoint_url From 8fb480bc5320dc5245c6301ffa3a24a5fe423fa3 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:45:24 -0400 Subject: [PATCH 11/30] Debugging publish --- .github/workflows/main-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main-publish.yml b/.github/workflows/main-publish.yml index 57847ecbb..560d5a489 100644 --- a/.github/workflows/main-publish.yml +++ b/.github/workflows/main-publish.yml @@ -28,7 +28,7 @@ jobs: with: python-version: 3.9 - name: Install Python dependencies for publish - run: pip install requests toml + run: pip boto3 install requests toml - name: Publish env: PYPI_USER: ${{ secrets.PYPI_USER }} From 3cd25c594087686fd9dd8c276e76c01d27df0fd4 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:50:18 -0400 Subject: [PATCH 12/30] Debugging publish --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1462fbe92..c03d46e93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.6.0.1b2" # TODO: To become 7.6.0 +version = "7.6.0.1b3" # TODO: To become 7.6.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From ebfa631e4f0fa3be351bdea998908f5d995bd427 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Thu, 29 Jun 2023 16:51:05 -0400 Subject: [PATCH 13/30] Debugging publish --- .github/workflows/main-publish.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main-publish.yml b/.github/workflows/main-publish.yml index 560d5a489..7a962cfca 100644 --- a/.github/workflows/main-publish.yml +++ b/.github/workflows/main-publish.yml @@ -28,7 +28,7 @@ jobs: with: python-version: 3.9 - name: Install Python dependencies for publish - run: pip boto3 install requests toml + run: pip install boto3 requests toml - name: Publish env: PYPI_USER: ${{ secrets.PYPI_USER }} diff --git a/pyproject.toml b/pyproject.toml index c03d46e93..4bd949aa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.6.0.1b3" # TODO: To become 7.6.0 +version = "7.6.0.1b4" # TODO: To become 7.6.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 024c6b4148dac2f5b167c066d1ee4d8ac5963be4 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 5 Jul 2023 12:36:48 -0400 Subject: [PATCH 14/30] Changed boto3 localstack monkeypatching to use LOCALSTACK_S3_URL and LOCALSTACK_SQS_URL rather than S3_URL and SQS_URL. --- CHANGELOG.rst | 5 +++-- dcicutils/boto_monkey_patching.py | 8 ++++---- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e465a199..331411805 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,8 +9,9 @@ Change Log 7.6.0 ===== * Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for - for S3 or SQS boto3 client/resource creation using the S3_URL or SQS_URL environment variables - to specify that these services should use a locally running ersatz instance of S3 or SQS via localstack. + for S3 or SQS boto3 client/resource creation using the LOCALSTACK_S3_URL or LOCALSTACK_SQS_URL + environment variables to specify that these services should use a locally running ersatz + instance of S3 or SQS via localstack. 7.5.2 diff --git a/dcicutils/boto_monkey_patching.py b/dcicutils/boto_monkey_patching.py index 32af607f2..4b9a4dc49 100644 --- a/dcicutils/boto_monkey_patching.py +++ b/dcicutils/boto_monkey_patching.py @@ -1,8 +1,8 @@ # Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url. # Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the # localstack utility, which provides a way to run some AWS services locally, for testing purposes. -# Currently only supported for S3 and SQS. To use this set the environment variables S3_URL and/or -# SQS_URL to the localstack URL, for example, http://localhost:4566. +# Currently only supported for S3 and SQS. To use this set the environment variables LOCALSTACK_S3_URL +# and/or LOCALSTACK_SQS_URL to the localstack URL, for example, http://localhost:4566. # Reference: https://localstack.cloud import boto3 @@ -13,8 +13,8 @@ _boto_client_original = boto3.client _boto_resource_original = boto3.resource _boto_service_overrides_supported = [ - {"service": "s3", "env": "S3_URL"}, - {"service": "sqs", "env": "SQS_URL"} + {"service": "s3", "env": "LOCALSTACK_S3_URL"}, + {"service": "sqs", "env": "LOCALSTACK_SQS_URL"} ] # This will entirely disable this feature; for troubleshooting only. _boto_monkey_patching_disabled = False diff --git a/pyproject.toml b/pyproject.toml index 4bd949aa0..0b380a463 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.6.0.1b4" # TODO: To become 7.6.0 +version = "7.6.0.1b5" # TODO: To become 7.6.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 7e2f6f0ec6afa5fa886b187a7fc6bf9f1c9c3da7 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 5 Jul 2023 13:04:04 -0400 Subject: [PATCH 15/30] Added some notes on localstack usage in getting_started.rst and added localstack to dev-dependencies in pyproject.toml. --- docs/source/getting_started.rst | 4 + poetry.lock | 436 +++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 440 insertions(+), 1 deletion(-) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 52914b886..9015f395f 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -30,3 +30,7 @@ Central metadata functions The most useful utilities functions for most users are the metadata functions, which generally are used to access, create, or edit object metadata on the Fourfront portal. Since this utilities module is a pip-installable Python package, they can be leveraged as an API to the portal in your scripts. All of these functions are contained within ``dcicutils.ff_utils.py``. See example usage of these functions `here <./examples.html#metadata>`_ + +Local development notes +^^^^^^^^^^^^^^^^^^^^^^^ +For local debugging and development, there is some support for the ``localstack`` package (https://localstack.cloud/), or something like it, to use a local ersatz version of certain AWS services, namely, currently, for S3 and SQS, whereby any boto3 based calls which use these services will optionally use the local ersatz version. To take advantage of this simply set the environment variables ``LOCALSTACK_S3_URL`` and/or ``LOCALSTACK_SQS_URL`` environment variables when running your process, whatever that may be, e.g. ``export LOCALSTACK_S3_URL=http://localhost:4566``. To install ``localstack`` use ``pip install localstack``, and to start it use ``localstack``. FYI this support is implemented via "monkey patching" hooked on these environment variables (see the ``boto_monkey_patching.py`` module for details). diff --git a/poetry.lock b/poetry.lock index 5bf042218..fa87bd3ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -419,6 +419,18 @@ files = [ [package.dependencies] typing-extensions = {version = "*", markers = "python_version < \"3.8\""} +[[package]] +name = "cachetools" +version = "5.0.0" +description = "Extensible memoizing collections and decorators" +category = "dev" +optional = false +python-versions = "~=3.7" +files = [ + {file = "cachetools-5.0.0-py3-none-any.whl", hash = "sha256:8fecd4203a38af17928be7b90689d8083603073622229ca7077b72d8e5a976e4"}, + {file = "cachetools-5.0.0.tar.gz", hash = "sha256:486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6"}, +] + [[package]] name = "certifi" version = "2022.12.7" @@ -593,6 +605,22 @@ files = [ {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, ] +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + [[package]] name = "colorama" version = "0.4.6" @@ -714,6 +742,54 @@ test = ["iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-co test-randomorder = ["pytest-randomly"] tox = ["tox"] +[[package]] +name = "dill" +version = "0.3.2" +description = "serialize all of python" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*" +files = [ + {file = "dill-0.3.2.zip", hash = "sha256:6e12da0d8e49c220e8d6e97ee8882002e624f1160289ce85ec2cc0a5246b3a2e"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + +[[package]] +name = "dnslib" +version = "0.9.23" +description = "Simple library to encode/decode DNS wire-format packets" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "dnslib-0.9.23-py2-none-any.whl", hash = "sha256:9eb851ac721eea51834d43795478ac9b48272c61ba97cc4a160668b50aff39ec"}, + {file = "dnslib-0.9.23-py3-none-any.whl", hash = "sha256:46137e8ef6ef52b24a16d47e0786a99dd103ab1e71eea616f21371accbccc557"}, + {file = "dnslib-0.9.23.tar.gz", hash = "sha256:310196d3e38ce2051b61eebbd2f1d08fcc934fa3360f22031864d16efe8bca77"}, +] + +[[package]] +name = "dnspython" +version = "2.3.0" +description = "DNS toolkit" +category = "dev" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] + +[package.extras] +curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] +dnssec = ["cryptography (>=2.6,<40.0)"] +doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] +doq = ["aioquic (>=0.9.20)"] +idna = ["idna (>=2.1,<4.0)"] +trio = ["trio (>=0.14,<0.23)"] +wmi = ["wmi (>=1.5.1,<2.0.0)"] + [[package]] name = "docker" version = "4.4.4" @@ -736,6 +812,25 @@ websocket-client = ">=0.32.0" ssh = ["paramiko (>=2.4.2)"] tls = ["cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] +[[package]] +name = "ecdsa" +version = "0.18.0" +description = "ECDSA cryptographic signature library (pure python)" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "ecdsa-0.18.0-py2.py3-none-any.whl", hash = "sha256:80600258e7ed2f16b9aa1d7c295bd70194109ad5a30fdee0eaeefef1d4c559dd"}, + {file = "ecdsa-0.18.0.tar.gz", hash = "sha256:190348041559e21b22a1d65cee485282ca11a6f81d503fddb84d5017e9ed1e49"}, +] + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +gmpy = ["gmpy"] +gmpy2 = ["gmpy2"] + [[package]] name = "elasticsearch" version = "7.13.4" @@ -890,6 +985,116 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "localstack" +version = "2.1.0" +description = "LocalStack - A fully functional local Cloud stack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-2.1.0.tar.gz", hash = "sha256:49213fa70ebcd9c9d40414a89c2cd5d0fe9a9a8e8e24cc6c9e917bdfbebe464c"}, +] + +[package.dependencies] +localstack-core = "*" +localstack-ext = "2.1.0" + +[package.extras] +full = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.1.0)"] +runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.1.0)"] + +[[package]] +name = "localstack-core" +version = "2.1.0" +description = "The core library and runtime of LocalStack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-core-2.1.0.tar.gz", hash = "sha256:f424b920148e53d2fa2599954cdaa8b8326c8d2d182e928b2a4f059f4c60e255"}, + {file = "localstack_core-2.1.0-py3-none-any.whl", hash = "sha256:bbd3548b04d0dfdb0e4eb369dc6118860a22a28fc9546af101cfcd3bafaec712"}, +] + +[package.dependencies] +cachetools = ">=5.0.0,<5.1.0" +click = ">=7.0" +cryptography = "*" +dill = "0.3.2" +dnspython = ">=1.16.0" +importlib-metadata = {version = "<5.0", markers = "python_version < \"3.8\""} +plux = ">=1.3.1" +psutil = ">=5.4.8,<6.0.0" +python-dotenv = ">=0.19.1" +pyyaml = ">=5.1" +requests = ">=2.20.0" +rich = ">=12.3.0" +semver = ">=2.10" +stevedore = ">=3.4.0" +tailer = ">=0.4.1" +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["Cython", "autoflake", "black (==22.3.0)", "coveralls (>=3.3.1)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "isort (==5.12.0)", "networkx (>=2.8.4)", "pandoc", "pre-commit (==2.13.0)", "pypandoc", "pyproject-flake8 (>=6.0.0.post1)", "rstr (>=3.2.0)"] +full = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.11.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.29.133)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (==0.14.2)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.10.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +runtime = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.11.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.29.133)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (==0.14.2)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.10.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +test = ["coverage[toml] (>=5.5)", "deepdiff (>=5.5.0)", "jsonpath-ng (>=1.5.3)", "pytest (==6.2.4)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)"] +typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]"] + +[[package]] +name = "localstack-ext" +version = "2.1.0" +description = "Extensions for LocalStack" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "localstack-ext-2.1.0.tar.gz", hash = "sha256:3102229d174650656b4b75639b55524f6dbfb7bc5604ecf18f09d9f834fcaf97"}, +] + +[package.dependencies] +dill = ">=0.3.2" +dnslib = ">=0.9.10" +dnspython = ">=1.16.0" +localstack-core = "2.1.0" +plux = ">=1.3.1" +pyaes = ">=1.6.0" +python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"]} +requests = ">=2.20.0" +tabulate = "*" +windows-curses = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +package = ["python-minifier"] +runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3,<0.10)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (==1.0.7)", "docker (>=6.0.1)", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (==21.7.0)", "localstack-core[runtime] (==2.1.0)", "localstack-plugin-persistence (>=0.1.3)", "moto-ext[all]", "paho-mqtt (>=1.5)", "parquet (>=1.3.1)", "parse (==1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.1.0)", "presto-python-client (>=0.7.0)", "pyftpdlib (>=1.5.6)", "pyhive[hive] (>=0.6.1)", "pyion2json (>=0.0.2)", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "srp-ext (==1.0.7.1)", "testing.common.database (>=1.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] +test = ["Js2Py (>=0.71)", "PyAthena[pandas]", "aiohttp", "aws_xray_sdk (>=2.4.2)", "awsiotsdk", "awswrangler (>=2.19.0)", "black (==22.3.0)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "gremlinpython", "isort (==5.12.0)", "jws (>=0.1.3)", "localstack-core[test] (==2.1.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "packaging (<22)", "paramiko (>=2.11.0,<2.12.0)", "portalocker", "pre-commit (==2.13.0)", "pyarrow", "pykafka", "pymongo", "pymssql", "pymysql", "pyproject-flake8 (==6.0.0.post1)", "pytest (==6.2.4)", "pytest-dependency (==0.5.1)", "pytest-httpserver (>=1.0.1)", "pytest-instafail (>=0.4.2)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)", "python-terraform", "redis (==4.5.4)", "redshift_connector", "stomp.py (==8.0.1)"] + +[[package]] +name = "markdown-it-py" +version = "2.2.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, + {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" +typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "mccabe" version = "0.7.0" @@ -902,6 +1107,18 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mirakuru" version = "2.5.1" @@ -954,6 +1171,18 @@ files = [ {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] +[[package]] +name = "pbr" +version = "5.11.1" +description = "Python Build Reasonableness" +category = "dev" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-5.11.1-py2.py3-none-any.whl", hash = "sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b"}, + {file = "pbr-5.11.1.tar.gz", hash = "sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3"}, +] + [[package]] name = "pluggy" version = "1.0.0" @@ -973,6 +1202,24 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "plux" +version = "1.3.2" +description = "A dynamic code loading framework for building pluggable Python distributions" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "plux-1.3.2-py3-none-any.whl", hash = "sha256:ac5762a278dfcdb3b747ecab6324b93fe3cdf70c93314b4c50931950340ecbb9"}, + {file = "plux-1.3.2.tar.gz", hash = "sha256:3862a76505779002b1edd9ee8d75c3524241052ccdad5277e9948dff62f1ba7d"}, +] + +[package.dependencies] +stevedore = ">=3.4" + +[package.extras] +dev = ["black (==21.6b0)", "isort (==5.9.1)", "pytest (==6.2.4)"] + [[package]] name = "port-for" version = "0.6.3" @@ -1012,6 +1259,29 @@ files = [ [package.extras] test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +[[package]] +name = "pyaes" +version = "1.6.1" +description = "Pure-Python Implementation of the AES block-cipher and common modes of operation" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pyaes-1.6.1.tar.gz", hash = "sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + [[package]] name = "pycodestyle" version = "2.9.1" @@ -1048,6 +1318,21 @@ files = [ {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, ] +[[package]] +name = "pygments" +version = "2.15.1" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, + {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pyjwt" version = "2.6.0" @@ -1199,6 +1484,44 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "0.21.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "python-dotenv-0.21.1.tar.gz", hash = "sha256:1c93de8f636cde3ce377292818d0e440b6e45a82f215c3744979151fa8151c49"}, + {file = "python_dotenv-0.21.1-py3-none-any.whl", hash = "sha256:41e12e0318bebc859fcc4d97d4db8d20ad21721a6aa5047dd59f090391cb549a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-jose" +version = "3.3.0" +description = "JOSE implementation in Python" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "python-jose-3.3.0.tar.gz", hash = "sha256:55779b5e6ad599c6336191246e95eb2293a9ddebd555f796a65f838f07e5d78a"}, + {file = "python_jose-3.3.0-py2.py3-none-any.whl", hash = "sha256:9b1376b023f8b298536eedd47ae1089bcdb848f1535ab30555cd92002d78923a"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryptography\""} +ecdsa = "!=0.15" +pyasn1 = "*" +rsa = "*" + +[package.extras] +cryptography = ["cryptography (>=3.4.0)"] +pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] +pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] + [[package]] name = "pytz" version = "2023.3" @@ -1330,6 +1653,41 @@ files = [ [package.extras] idna2008 = ["idna"] +[[package]] +name = "rich" +version = "13.4.2" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "dev" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"}, + {file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +category = "dev" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "s3transfer" version = "0.6.0" @@ -1348,6 +1706,18 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "semver" +version = "3.0.1" +description = "Python helper for Semantic Versioning (https://semver.org)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, + {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1384,6 +1754,22 @@ files = [ {file = "soupsieve-2.4.tar.gz", hash = "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a"}, ] +[[package]] +name = "stevedore" +version = "3.5.2" +description = "Manage dynamic plugins for Python applications" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "stevedore-3.5.2-py3-none-any.whl", hash = "sha256:fa2630e3d0ad3e22d4914aff2501445815b9a4467a6edc49387c667a38faf5bf"}, + {file = "stevedore-3.5.2.tar.gz", hash = "sha256:cf99f41fc0d5a4f185ca4d3d42b03be9011b0a1ec1a4ea1a282be1b4b306dcc2"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} +pbr = ">=2.0.0,<2.1.0 || >2.1.0" + [[package]] name = "structlog" version = "19.2.0" @@ -1405,6 +1791,32 @@ dev = ["coverage", "freezegun (>=0.2.8)", "pre-commit", "pretend", "pytest (>=3. docs = ["sphinx", "twisted"] tests = ["coverage", "freezegun (>=0.2.8)", "pretend", "pytest (>=3.3.0)", "python-rapidjson", "simplejson"] +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "tailer" +version = "0.4.1" +description = "Python tail is a simple implementation of GNU tail and head." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "tailer-0.4.1.tar.gz", hash = "sha256:78d60f23a1b8a2d32f400b3c8c06b01142ac7841b75d8a1efcb33515877ba531"}, +] + [[package]] name = "toml" version = "0.10.2" @@ -1550,6 +1962,28 @@ WebOb = ">=1.2" docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.8)"] tests = ["PasteDeploy", "WSGIProxy2", "coverage", "mock", "nose (<1.3.0)", "pyquery"] +[[package]] +name = "windows-curses" +version = "2.3.1" +description = "Support for the standard curses module on Windows" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "windows_curses-2.3.1-cp310-cp310-win32.whl", hash = "sha256:2644f4547ae5124ce5129b66faa59ee0995b7b7205ed5e3920f6ecfef2e46275"}, + {file = "windows_curses-2.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b541520895649c0835771775034a2b4edf36da3c3d9381c5022b5b4f9a5014e"}, + {file = "windows_curses-2.3.1-cp311-cp311-win32.whl", hash = "sha256:25e7ff3d77aed6c747456b06fbc1528d67fc59d1ef3be9ca244774e65e6bdbb2"}, + {file = "windows_curses-2.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:395656bfe88d6f60cb18604605d423e0f2d1c3a8f550507dca5877a9d0b3a0f3"}, + {file = "windows_curses-2.3.1-cp36-cp36m-win32.whl", hash = "sha256:6ea8e1c4536fee248ee3f88e5010871df749932b7e829e2f012e5d23bd2fe31d"}, + {file = "windows_curses-2.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59856b41676c4b3eb527eb6b1478803d4dc92413b2e63aea762407807ffcd3ac"}, + {file = "windows_curses-2.3.1-cp37-cp37m-win32.whl", hash = "sha256:9cd0ba6efde23930736eff45a0aa0af6fd82e60b4787a46157ef4956d2c52b06"}, + {file = "windows_curses-2.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f9a7fcd03934e40238f9bbeddae51e3fdc442f28bca50afccdc521245ed39439"}, + {file = "windows_curses-2.3.1-cp38-cp38-win32.whl", hash = "sha256:5c55ebafdb402cfa927174a03d651cd1b1e76d6e6cf71818f9d3378636c00e74"}, + {file = "windows_curses-2.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:a551aaa09d6ec28f64ade8e85fd0c52880c8e9114729a79c34803104e49bed71"}, + {file = "windows_curses-2.3.1-cp39-cp39-win32.whl", hash = "sha256:aab7e28133bf81769cddf8b3c3c8ab89e76cd43effd371c6370e918b6dfccf1b"}, + {file = "windows_curses-2.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:85675de4ae7058348140daae83a8a7b81147a84ef9ab699307b3168f9490292f"}, +] + [[package]] name = "zipp" version = "3.15.0" @@ -1569,4 +2003,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.10" -content-hash = "e1fc871f703b88fae9d54c773ebeac0285757df89eb0dca918c38ff0969b3525" +content-hash = "7627ed3c48e8ff61f8c9da056b5c0541b49e45a551cb6113cc74ba92dc7d1afa" diff --git a/pyproject.toml b/pyproject.toml index 0b380a463..5e8db9261 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ coverage = ">=7.2.3" # coveralls = ">=3.3.1" flake8 = ">=3.9.2" flaky = ">=3.7.0" +localstack = "^2.1.0" pytest = ">=4.5.0" pytest-cov = ">=2.7.1" pytest-mock = ">=1.11.0" From 11d02095ccf62315ca9ab1256866435df9c7502b Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 5 Jul 2023 13:35:53 -0400 Subject: [PATCH 16/30] Added some notes on localstack usage in getting_started.rst and added localstack to dev-dependencies in pyproject.toml. --- dcicutils/boto_monkey_patching.py | 16 +++++++++++++--- test/conftest.py | 10 +++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/dcicutils/boto_monkey_patching.py b/dcicutils/boto_monkey_patching.py index 4b9a4dc49..d239ec2c9 100644 --- a/dcicutils/boto_monkey_patching.py +++ b/dcicutils/boto_monkey_patching.py @@ -9,22 +9,32 @@ import os from typing import Optional +LOCALSTACK_S3_URL_ENVIRON_NAME = "LOCALSTACK_S3_URL" +LOCALSTACK_SQS_URL_ENVIRON_NAME = "LOCALSTACK_SQS_URL" + _boto_client_original = boto3.client _boto_resource_original = boto3.resource _boto_service_overrides_supported = [ - {"service": "s3", "env": "LOCALSTACK_S3_URL"}, - {"service": "sqs", "env": "LOCALSTACK_SQS_URL"} + {"service": "s3", "env": LOCALSTACK_S3_URL_ENVIRON_NAME}, + {"service": "sqs", "env": LOCALSTACK_SQS_URL_ENVIRON_NAME} ] + # This will entirely disable this feature; for troubleshooting only. _boto_monkey_patching_disabled = False -# For import only in test_boto_monkey_patching. +# For import only in test_boto_monkey_patching; +# the list of AWS services for which we support this monkey patching facility (e.g. ["s3", "sqs"]). _boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported] # For import only in test_boto_monkey_patching. def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]: + """ + For the given AWS service name (e.g. "s3" or "sqs") returns environment variable name which + needs to be set in order to use a different (e.g. localstack version of the) endpoint URL + when creating a boto3 client or resource. E.g. given "s3" this will return "LOCALSTACK_S3_URL". + """ for item in _boto_service_overrides_supported: if item["service"] == service: return item["env"] diff --git a/test/conftest.py b/test/conftest.py index b59d69aee..abac63e67 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,10 @@ import os -# Disable any AWS endpoint-url environment variables overrides for testing. -if "S3_URL" in os.environ: - os.environ.pop("S3_URL") -if "SQS_URL" in os.environ: - os.environ.pop("SQS_URL") +# Disable any AWS endpoint-url (ala boto_monkey_patching/localstack) environment variables overrides for testing. +if "LOCALSTACK_S3_URL" in os.environ: + os.environ.pop("LOCALSTACK_S3_URL") +if "LOCALSTACK_SQS_URL" in os.environ: + os.environ.pop("LOCALSTACK_SQS_URL") import pytest import requests From fd1ed9a6d4e2fe6bd1e313ee3e4bcf44f426fd4e Mon Sep 17 00:00:00 2001 From: David Michaels Date: Wed, 5 Jul 2023 14:20:57 -0400 Subject: [PATCH 17/30] Fix for (smaht) for test_common.py - this fix needs to go in master too --- test/test_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_common.py b/test/test_common.py index 55054a4e6..35e096256 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -1,9 +1,9 @@ -from dcicutils.common import EnvName, OrchestratedApp, APP_CGAP, APP_FOURFRONT, ORCHESTRATED_APPS +from dcicutils.common import EnvName, OrchestratedApp, APP_CGAP, APP_FOURFRONT, APP_SMAHT, ORCHESTRATED_APPS def test_app_constants(): - assert set(ORCHESTRATED_APPS) == {APP_CGAP, APP_FOURFRONT} == {'cgap', 'fourfront'} + assert set(ORCHESTRATED_APPS) == {APP_CGAP, APP_FOURFRONT, APP_SMAHT} == {'cgap', 'fourfront', 'smaht'} # For thexe next two, which are really type hints, just test that they exist. assert EnvName From 11d881c4d972a4514413217fd5852e20464cc1a3 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Mon, 7 Aug 2023 14:00:10 -0400 Subject: [PATCH 18/30] merge from master --- poetry.lock | 3 --- 1 file changed, 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index c84cc3be6..5da3809e1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1263,7 +1263,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "ptable" version = "0.9.2" description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" ->>>>>>> master category = "dev" optional = false python-versions = "*" @@ -1281,9 +1280,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, -======= {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, ->>>>>>> master ] [[package]] From f537d4441a4d5d69b23ce9caa89d26a47f333b56 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Mon, 7 Aug 2023 14:02:53 -0400 Subject: [PATCH 19/30] merge from master --- poetry.lock | 87 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5da3809e1..cc950c88f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "async-timeout" @@ -588,14 +588,14 @@ files = [ [[package]] name = "click" -version = "8.1.3" +version = "8.1.6" description = "Composable command line interface toolkit" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, ] [package.dependencies] @@ -981,33 +981,33 @@ files = [ [[package]] name = "localstack" -version = "2.1.0" +version = "2.2.0" description = "LocalStack - A fully functional local Cloud stack" category = "dev" optional = false python-versions = "*" files = [ - {file = "localstack-2.1.0.tar.gz", hash = "sha256:49213fa70ebcd9c9d40414a89c2cd5d0fe9a9a8e8e24cc6c9e917bdfbebe464c"}, + {file = "localstack-2.2.0.tar.gz", hash = "sha256:3b3ed3d4e13fff791ebcc8d9e58617b7ba62b87ae3a6925645050134b1643e2d"}, ] [package.dependencies] localstack-core = "*" -localstack-ext = "2.1.0" +localstack-ext = "2.2.0" [package.extras] -full = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.1.0)"] -runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.1.0)"] +full = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.2.0)"] +runtime = ["localstack-core[runtime]", "localstack-ext[runtime] (==2.2.0)"] [[package]] name = "localstack-core" -version = "2.1.0" +version = "2.2.0" description = "The core library and runtime of LocalStack" category = "dev" optional = false python-versions = "*" files = [ - {file = "localstack-core-2.1.0.tar.gz", hash = "sha256:f424b920148e53d2fa2599954cdaa8b8326c8d2d182e928b2a4f059f4c60e255"}, - {file = "localstack_core-2.1.0-py3-none-any.whl", hash = "sha256:bbd3548b04d0dfdb0e4eb369dc6118860a22a28fc9546af101cfcd3bafaec712"}, + {file = "localstack-core-2.2.0.tar.gz", hash = "sha256:cc46a5e585f4265e9273439335ef1fa2f4b9163f157145d298b62d388f75fb60"}, + {file = "localstack_core-2.2.0-py3-none-any.whl", hash = "sha256:34b41b22da4134089ab9bdec71049d5a7cb45ba58d30e62545285b54cd4b3629"}, ] [package.dependencies] @@ -1030,29 +1030,30 @@ typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] dev = ["Cython", "autoflake", "black (==22.3.0)", "coveralls (>=3.3.1)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "isort (==5.12.0)", "networkx (>=2.8.4)", "pandoc", "pre-commit (==2.13.0)", "pypandoc", "pyproject-flake8 (>=6.0.0.post1)", "rstr (>=3.2.0)"] -full = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.11.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.29.133)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (==0.14.2)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.10.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] -runtime = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.11.1)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.29.133)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (==0.14.2)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.10.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +full = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.12.0)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.31.2)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.13.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] +runtime = ["Quart (>=0.18)", "Werkzeug (>=2.3.4)", "airspeed-ext (==0.5.19)", "amazon-kclpy (>=2.0.6,!=2.1.0)", "antlr4-python3-runtime (==4.12.0)", "apispec (>=5.1.1)", "aws-sam-translator (>=1.15.1)", "awscli (>=1.22.90)", "awscrt (>=0.13.14)", "boto (>=2.49.0)", "boto3 (>=1.26.121)", "botocore (>=1.31.2)", "cbor2 (>=5.2.0)", "crontab (>=0.22.6)", "dnslib (>=0.9.10)", "dnspython (>=1.16.0)", "docker (==6.0.1)", "flask (>=2.3.2)", "flask-cors (>=3.0.3,<3.1.0)", "flask-swagger (==0.2.12)", "hypercorn (>=0.14.4)", "json5 (==0.9.11)", "jsonpatch (>=1.24,<2.0)", "jsonpath-ng (>=1.5.3)", "jsonpath-rw (>=1.4.0,<2.0.0)", "localstack-client (>=2.0)", "moto-ext[all] (==4.1.13.post1)", "opensearch-py (==2.1.1)", "pproxy (>=2.7.0)", "pymongo (>=4.2.0)", "pyopenssl (>=23.0.0)", "readerwriterlock (>=1.0.7)", "requests-aws4auth (>=1.0)", "vosk (==0.3.43)", "xmltodict (>=0.11.0)"] test = ["coverage[toml] (>=5.5)", "deepdiff (>=5.5.0)", "jsonpath-ng (>=1.5.3)", "pytest (==6.2.4)", "pytest-httpserver (>=1.0.1)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)"] -typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]"] +typehint = ["boto3-stubs[acm,amplify,apigateway,apigatewayv2,appconfig,appsync,athena,autoscaling,backup,batch,ce,cloudcontrol,cloudformation,cloudfront,cloudtrail,cloudwatch,codecommit,cognito-identity,cognito-idp,dms,docdb,dynamodb,dynamodbstreams,ec2,ecr,ecs,efs,eks,elasticache,elasticbeanstalk,elbv2,emr,emr-serverless,es,events,firehose,fis,glacier,glue,iam,iot,iot-data,iotanalytics,iotwireless,kafka,kinesis,kinesisanalytics,kinesisanalyticsv2,kms,lakeformation,lambda,logs,mediaconvert,mediastore,mq,mwaa,neptune,opensearch,organizations,pi,qldb,qldb-session,rds,rds-data,redshift,redshift-data,resource-groups,resourcegroupstaggingapi,route53,route53resolver,s3,s3control,sagemaker,sagemaker-runtime,secretsmanager,serverlessrepo,servicediscovery,ses,sesv2,sns,sqs,ssm,stepfunctions,sts,timestream-query,timestream-write,transcribe,xray]"] [[package]] name = "localstack-ext" -version = "2.1.0" +version = "2.2.0" description = "Extensions for LocalStack" category = "dev" optional = false python-versions = "*" files = [ - {file = "localstack-ext-2.1.0.tar.gz", hash = "sha256:3102229d174650656b4b75639b55524f6dbfb7bc5604ecf18f09d9f834fcaf97"}, + {file = "localstack-ext-2.2.0.tar.gz", hash = "sha256:04b2b8d5345a60db6979e0de199865be7be44305a8d2e181d7d837e30691a859"}, ] [package.dependencies] dill = ">=0.3.2" dnslib = ">=0.9.10" dnspython = ">=1.16.0" -localstack-core = "2.1.0" +localstack-core = "2.2.0" plux = ">=1.3.1" pyaes = ">=1.6.0" +python-dateutil = ">=2.8" python-jose = {version = ">=3.1.0,<4.0.0", extras = ["cryptography"]} requests = ">=2.20.0" tabulate = "*" @@ -1060,8 +1061,8 @@ windows-curses = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] package = ["python-minifier"] -runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3,<0.10)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (==1.0.7)", "docker (>=6.0.1)", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (==21.7.0)", "localstack-core[runtime] (==2.1.0)", "localstack-plugin-persistence (>=0.1.3)", "moto-ext[all]", "paho-mqtt (>=1.5)", "parquet (>=1.3.1)", "parse (==1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.1.0)", "presto-python-client (>=0.7.0)", "pyftpdlib (>=1.5.6)", "pyhive[hive] (>=0.6.1)", "pyion2json (>=0.0.2)", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "srp-ext (==1.0.7.1)", "testing.common.database (>=1.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] -test = ["Js2Py (>=0.71)", "PyAthena[pandas]", "aiohttp", "aws_xray_sdk (>=2.4.2)", "awsiotsdk", "awswrangler (>=2.19.0)", "black (==22.3.0)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "gremlinpython", "isort (==5.12.0)", "jws (>=0.1.3)", "localstack-core[test] (==2.1.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "packaging (<22)", "paramiko (>=2.11.0,<2.12.0)", "portalocker", "pre-commit (==2.13.0)", "pyarrow", "pykafka", "pymongo", "pymssql", "pymysql", "pyproject-flake8 (==6.0.0.post1)", "pytest (==6.2.4)", "pytest-dependency (==0.5.1)", "pytest-httpserver (>=1.0.1)", "pytest-instafail (>=0.4.2)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)", "python-terraform", "redis (==4.5.4)", "redshift_connector", "stomp.py (==8.0.1)"] +runtime = ["Js2Py (>=0.71)", "PyJWT (>=1.7.0)", "Whoosh (>=2.7.4)", "amazon.ion (>=0.9.3,<0.10)", "avro (>=1.11.0)", "aws-encryption-sdk (>=3.1.0)", "dirtyjson (==1.0.7)", "docker (>=6.0.1)", "docker-registry-client (>=0.5.2)", "dulwich (>=0.19.16)", "graphql-core (>=3.0.3)", "janus (>=0.5.0)", "jsonpatch (>=1.32)", "kafka-python", "kubernetes (==21.7.0)", "localstack-core[runtime] (==2.2.0)", "localstack-plugin-persistence (>=0.1.3)", "moto-ext[all]", "paho-mqtt (>=1.5)", "parquet (>=1.3.1)", "parse (==1.19.0)", "pg8000 (>=1.10)", "postgres (>=2.2.2)", "postgresql-proxy (>=0.1.0)", "presto-python-client (>=0.7.0)", "pyftpdlib (>=1.5.6)", "pyhive[hive] (>=0.6.1)", "pyion2json (>=0.0.2)", "pymysql", "pyqldb (>=3.2,<4.0)", "python-snappy (>=0.6)", "readerwriterlock (>=1.0.7)", "rsa (>=4.0)", "sql-metadata (>=2.6.0)", "srp-ext (==1.0.7.1)", "testing.common.database (>=1.1.0)", "tornado (>=6.0)", "warrant-ext (>=0.6.2)", "websockets (>=8.1)"] +test = ["Js2Py (>=0.71)", "PyAthena[pandas]", "aiohttp", "aws_xray_sdk (>=2.4.2)", "awsiotsdk", "awswrangler (>=2.19.0)", "black (==22.3.0)", "coverage[toml] (>=5.0.0)", "coveralls (>=3.3.1)", "deepdiff (>=5.5.0)", "flake8 (>=6.0.0)", "flake8-black (>=0.3.6)", "flake8-isort (>=6.0.0)", "flake8-quotes (>=3.3.2)", "gremlinpython", "isort (==5.12.0)", "jws (>=0.1.3)", "localstack-core[test] (==2.2.0)", "msal", "msal-extensions", "msrest", "mysql-connector-python", "neo4j", "nest-asyncio (>=1.4.1)", "packaging (<22)", "paramiko (>=2.11.0,<2.12.0)", "portalocker", "pre-commit (==2.13.0)", "pyarrow", "pykafka", "pymongo", "pymssql", "pyproject-flake8 (==6.0.0.post1)", "pytest (==6.2.4)", "pytest-dependency (==0.5.1)", "pytest-httpserver (>=1.0.1)", "pytest-instafail (>=0.4.2)", "pytest-rerunfailures (==10.0)", "pytest-split (>=0.8.0)", "python-terraform", "redis (==4.5.4)", "redshift_connector", "stomp.py (==8.0.1)"] [[package]] name = "markdown-it-py" @@ -1165,6 +1166,18 @@ files = [ {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] +[[package]] +name = "pbr" +version = "5.11.1" +description = "Python Build Reasonableness" +category = "dev" +optional = false +python-versions = ">=2.6" +files = [ + {file = "pbr-5.11.1-py2.py3-none-any.whl", hash = "sha256:567f09558bae2b3ab53cb3c1e2e33e726ff3338e7bae3db5dc954b3a44eef12b"}, + {file = "pbr-5.11.1.tar.gz", hash = "sha256:aefc51675b0b533d56bb5fd1c8c6c0522fe31896679882e1c4c63d5e4a0fccb3"}, +] + [[package]] name = "pip-licenses" version = "3.5.5" @@ -1204,21 +1217,21 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "plux" -version = "1.3.2" +version = "1.4.0" description = "A dynamic code loading framework for building pluggable Python distributions" category = "dev" optional = false python-versions = "*" files = [ - {file = "plux-1.3.2-py3-none-any.whl", hash = "sha256:ac5762a278dfcdb3b747ecab6324b93fe3cdf70c93314b4c50931950340ecbb9"}, - {file = "plux-1.3.2.tar.gz", hash = "sha256:3862a76505779002b1edd9ee8d75c3524241052ccdad5277e9948dff62f1ba7d"}, + {file = "plux-1.4.0-py3-none-any.whl", hash = "sha256:9ee34e9a339a2f512db69fc2592eaabedbccd6280102b06a74c32ec5268b1916"}, + {file = "plux-1.4.0.tar.gz", hash = "sha256:4a432e2bd0fd50eb165781fd2a885e2169492f1f699429afda8130a3efdd4590"}, ] [package.dependencies] stevedore = ">=3.4" [package.extras] -dev = ["black (==21.6b0)", "isort (==5.9.1)", "pytest (==6.2.4)"] +dev = ["black (==22.3.0)", "isort (==5.9.1)", "pytest (==6.2.4)"] [[package]] name = "port-for" @@ -1266,6 +1279,17 @@ description = "A simple Python library for easily displaying tabular data in a v category = "dev" optional = false python-versions = "*" +files = [ + {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, +] + +[[package]] +name = "pyaes" +version = "1.6.1" +description = "Pure-Python Implementation of the AES block-cipher and common modes of operation" +category = "dev" +optional = false +python-versions = "*" files = [ {file = "pyaes-1.6.1.tar.gz", hash = "sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"}, ] @@ -1280,7 +1304,6 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, - {file = "PTable-0.9.2.tar.gz", hash = "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292"}, ] [[package]] @@ -1321,14 +1344,14 @@ files = [ [[package]] name = "pygments" -version = "2.15.1" +version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] @@ -1658,14 +1681,14 @@ idna2008 = ["idna"] [[package]] name = "rich" -version = "13.4.2" +version = "13.5.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "dev" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"}, - {file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"}, + {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, + {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, ] [package.dependencies] @@ -2006,4 +2029,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.10" -content-hash = "b8d6612bb28cfb9da79306a82b2ac35a20678e1f62ef86c93b8af3c3d1ed798e" +content-hash = "e27365e51b9935094f0da68ad3f0031c07bbdc24787d7bbc6155977fd737797f" From 3b7f2122c71e4fda5e9832ba92a320e1fe88789a Mon Sep 17 00:00:00 2001 From: David Michaels Date: Mon, 7 Aug 2023 14:15:08 -0400 Subject: [PATCH 20/30] merge from master --- poetry.lock | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index cc950c88f..185be6795 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1584,41 +1584,25 @@ files = [ [[package]] name = "pyyaml" -version = "5.4.1" +version = "5.3.1" description = "YAML parser and emitter for Python" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = "*" files = [ - {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, - {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, - {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, - {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, - {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, - {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, - {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, - {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, - {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, - {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, - {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, - {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, - {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, - {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, - {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, - {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, - {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] [[package]] @@ -2029,4 +2013,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.10" -content-hash = "e27365e51b9935094f0da68ad3f0031c07bbdc24787d7bbc6155977fd737797f" +content-hash = "a1942deff0d1a07a676b9f7753b8127a62a5179746bb519a624ce44343247641" From 2ae8317e03fd390ad43af06b9cb7a4679b958073 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Tue, 8 Aug 2023 15:41:57 -0400 Subject: [PATCH 21/30] Added localstack-ext (Apache-2.0) in license exception table. --- dcicutils/license_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dcicutils/license_utils.py b/dcicutils/license_utils.py index 13b8e40e9..0292976a7 100644 --- a/dcicutils/license_utils.py +++ b/dcicutils/license_utils.py @@ -862,6 +862,10 @@ class C4InfrastructureLicenseChecker(LicenseChecker): 'typed-function', # LICENSE at https://www.npmjs.com/package/typed-function?activeTab=code ], + + 'Other/Proprietary License': [ + 'localstack-ext' + ] } From d0f5a5a33a48026ec86f8616c04c60a47407a251 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 11 Aug 2023 10:44:27 -0400 Subject: [PATCH 22/30] typo --- dcicutils/license_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcicutils/license_utils.py b/dcicutils/license_utils.py index 0f3d22cbe..38369648c 100644 --- a/dcicutils/license_utils.py +++ b/dcicutils/license_utils.py @@ -878,7 +878,7 @@ class C4InfrastructureLicenseChecker(LicenseChecker): # This is known to be offered under Apache-2.0 license. # Ref: https://github.com/localstack/localstack/blob/master/LICENSE.txt 'localstack-ext' - ] + ], 'UNLICENSED': [ # The udn-browser library is our own and has been observed to sometimes show up in some contexts From daad33e45819e2582fd5f4d2d456620aa44071f4 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 11 Aug 2023 10:52:20 -0400 Subject: [PATCH 23/30] white space fix --- dcicutils/license_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcicutils/license_utils.py b/dcicutils/license_utils.py index 38369648c..be472df6a 100644 --- a/dcicutils/license_utils.py +++ b/dcicutils/license_utils.py @@ -876,7 +876,7 @@ class C4InfrastructureLicenseChecker(LicenseChecker): 'Other/Proprietary License': [ # This is known to be offered under Apache-2.0 license. - # Ref: https://github.com/localstack/localstack/blob/master/LICENSE.txt + # Ref: https://github.com/localstack/localstack/blob/master/LICENSE.txt 'localstack-ext' ], From 2a84d1805448f0262983a88563d2802e4f3bc7a5 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Sat, 12 Aug 2023 12:41:36 -0400 Subject: [PATCH 24/30] Minor comment/typo cleanup in publish_to_pypi.py --- CHANGELOG.rst | 1 + dcicutils/scripts/publish_to_pypi.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ef024343..e901449d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,7 @@ Change Log for S3 or SQS boto3 client/resource creation using the LOCALSTACK_S3_URL or LOCALSTACK_SQS_URL environment variables to specify that these services should use a locally running ersatz instance of S3 or SQS via localstack. +* Comment and typo cleanup in ``publish_to_pypi.py``. 7.7.2 diff --git a/dcicutils/scripts/publish_to_pypi.py b/dcicutils/scripts/publish_to_pypi.py index a454782b2..65cfde7bc 100644 --- a/dcicutils/scripts/publish_to_pypi.py +++ b/dcicutils/scripts/publish_to_pypi.py @@ -5,12 +5,13 @@ # 1. The git repo MUST NOT contain unstaged changes. # 2. The git repo MUST NOT contain staged but uncommitted changes. # 3. The git repo MUST NOT contain committed but unpushed changes. -# 4. The git repo package directories MUST NOT contain untracked files, +# 4. The git repo MUST be tagged (its most recent commit must be for a tag). +# 5. The git repo package directories MUST NOT contain untracked files, # OR if they do contain untracked files then you must confirm this is OK. -# 5. The version being published must NOT have already been published. +# 6. The version being published must NOT have already been published. # # ASSUMES you have these credentials environment variables correctly set for PyPi publishing; -# although a --username and --password are also supported to set these via command-line. +# although a --username and --password are alternatively supported to set these via command-line. # # - PYPI_USER # - PYPI_PASSWORD @@ -22,12 +23,9 @@ # option to skip this confimation, however it is only allowed when running in the # context of GitHub actions - it checks for the GITHUB_ACTIONS environment variable. # -# Prints warning if PY -# -# FYI: This was created late April 2023 after a junk file containing development -# logging output containing passwords was accidentally published to PyPi; -# item #4 above specifically addresses/prevents this. Perhaps better -# would be if publishing only happened via GitHub actions. +# FYI: This was created late April 2023 after a junk file containing development logging +# output containing passwords was accidentally published to PyPi; item #5 above specifically +# addresses/prevents this. Perhaps better would be if publishing only happened via GitHub actions. import argparse import os @@ -111,9 +109,9 @@ def publish_package(pypi_username: str = None, pypi_password: str = None, force_ if pypi_username != PYPI_API_TOKEN_USERNAME: if not force_allow_username: # Just in case someone really really needs this we will allow for now: --force-allow-username - ERROR_PRINT(f"Publishing with username/pasword is no longer allowed; must use API token instead.") + ERROR_PRINT(f"Publishing with username/password is no longer allowed; must use API token instead.") return False - WARNING_PRINT(f"Publishing with username/pasword is NOT recommmended; use API token instead;" + WARNING_PRINT(f"Publishing with username/password is NOT recommmended; use API token instead;" f"only allowing because you said: --force-allow-username") poetry_publish_command = [ "poetry", "publish", From fc91806d37a2f6486bb32553287bae973222dfbf Mon Sep 17 00:00:00 2001 From: David Michaels Date: Sun, 13 Aug 2023 12:38:31 -0400 Subject: [PATCH 25/30] Update PyYAML to ^6.0.1; Mac M1 with Python 3.9 requires 5.3.1 (not 5.4.1) or 6 and up. --- poetry.lock | 59 ++++++++++++++++++++++++++++++++++++-------------- pyproject.toml | 2 +- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/poetry.lock b/poetry.lock index 185be6795..31102b044 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1584,25 +1584,52 @@ files = [ [[package]] name = "pyyaml" -version = "5.3.1" +version = "6.0.1" description = "YAML parser and emitter for Python" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, - {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, - {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, - {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, - {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, - {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, - {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, - {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, - {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, - {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, - {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, - {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, - {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] @@ -2013,4 +2040,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.10" -content-hash = "a1942deff0d1a07a676b9f7753b8127a62a5179746bb519a624ce44343247641" +content-hash = "5d652586eef7475ad10cf40bb197feb7b14b77736f2f6431ddca25cf29decd8c" diff --git a/pyproject.toml b/pyproject.toml index 4b922f8ab..0c5c2e0c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ aws-requests-auth = ">=0.4.2,<1" docker = "^4.4.4" gitpython = "^3.1.2" pytz = ">=2020.4" -PyYAML = ">=5.1,<5.4" +PyYAML = "^6.0.1" requests = "^2.21.0" rfc3986 = "^1.4.0" structlog = "^19.2.0" From 7387f0fe5f0bc258aa555c45e034ad5ef95360ac Mon Sep 17 00:00:00 2001 From: David Michaels Date: Sun, 13 Aug 2023 12:47:57 -0400 Subject: [PATCH 26/30] Comments --- CHANGELOG.rst | 5 +++-- dcicutils/scripts/publish_to_pypi.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e901449d0..0143d0bdf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,10 +10,11 @@ Change Log ===== * Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for - for S3 or SQS boto3 client/resource creation using the LOCALSTACK_S3_URL or LOCALSTACK_SQS_URL + S3 or SQS boto3 client/resource creation using the ``LOCALSTACK_S3_URL`` or ``LOCALSTACK_SQS_URL`` environment variables to specify that these services should use a locally running ersatz - instance of S3 or SQS via localstack. + instance of S3 or SQS via localstack (https://localstack.cloud). * Comment and typo cleanup in ``publish_to_pypi.py``. +* Updates PyYAML to ^6.0.1 because Mac M1 (with Python 3.9) only likes 5.3.1 (not 5.4.1) or 6+. 7.7.2 diff --git a/dcicutils/scripts/publish_to_pypi.py b/dcicutils/scripts/publish_to_pypi.py index 65cfde7bc..1bcefe9e1 100644 --- a/dcicutils/scripts/publish_to_pypi.py +++ b/dcicutils/scripts/publish_to_pypi.py @@ -32,7 +32,6 @@ import requests import subprocess import toml - from typing import Tuple, Union From 646a265cdc1992bf108a5b8e4e89d01057528862 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 25 Aug 2023 07:43:08 -0400 Subject: [PATCH 27/30] CHANGELOG.rst updates. --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f84966446..b21230d13 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,6 +36,7 @@ Change Log for conceptual compatibility with ``get_schemas``. * Minor tweaks to ``dump_results_to_json`` for style reasons, and repairs to its overly complex and error-prone unit test. + * Fix to license_utils.py for localstack-ext. 7.8.0 From 0f3d3f07e7ffe4f3df34e5f65e4a49bad1791a3a Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 25 Aug 2023 13:28:31 -0400 Subject: [PATCH 28/30] Fix to test_s3_utils from kmp_sheet_utils_schema_hinting branch. --- pyproject.toml | 2 +- test/test_s3_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f349f1aad..76b630289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "7.9.0.1b1" # TODO: To become 7.10.0 +version = "7.9.0.1b2" # TODO: To become 7.10.0 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/test/test_s3_utils.py b/test/test_s3_utils.py index 3cfd05c7e..30fa207c3 100644 --- a/test/test_s3_utils.py +++ b/test/test_s3_utils.py @@ -409,7 +409,7 @@ def test_s3utils_get_google_key(): keys = s3u.get_google_key() assert isinstance(keys, dict) assert keys['type'] == 'service_account' - assert keys["project_id"] == "fourdn-fourfront" + assert keys["project_id"] == "fourfront-396315" # yes, this is a magic constant for dict_key in ['private_key_id', 'private_key', 'client_email', 'client_id', 'auth_uri', 'client_x509_cert_url']: assert keys[dict_key] From 6eecccf6df1b115a68ead13575ae71421c4e3a81 Mon Sep 17 00:00:00 2001 From: David Michaels Date: Fri, 25 Aug 2023 13:33:07 -0400 Subject: [PATCH 29/30] flake8 update --- test/test_s3_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_s3_utils.py b/test/test_s3_utils.py index 30fa207c3..0e98959cd 100644 --- a/test/test_s3_utils.py +++ b/test/test_s3_utils.py @@ -409,7 +409,7 @@ def test_s3utils_get_google_key(): keys = s3u.get_google_key() assert isinstance(keys, dict) assert keys['type'] == 'service_account' - assert keys["project_id"] == "fourfront-396315" # yes, this is a magic constant + assert keys["project_id"] == "fourfront-396315" # yes, this is a magic constant for dict_key in ['private_key_id', 'private_key', 'client_email', 'client_id', 'auth_uri', 'client_x509_cert_url']: assert keys[dict_key] From 8f7cb755329a9dc66386aa50ae8d5316fd79f93a Mon Sep 17 00:00:00 2001 From: David Michaels Date: Tue, 5 Sep 2023 11:09:03 -0400 Subject: [PATCH 30/30] comment --- dcicutils/captured_output.py | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 dcicutils/captured_output.py diff --git a/dcicutils/captured_output.py b/dcicutils/captured_output.py new file mode 100644 index 000000000..e5db67c42 --- /dev/null +++ b/dcicutils/captured_output.py @@ -0,0 +1,69 @@ +from collections import namedtuple +from contextlib import contextmanager +import io +import sys +from typing import Optional + +_real_stdout = sys.stdout +_real_stderr = sys.stderr + +@contextmanager +def captured_output(capture: bool = True): + """ + Context manager to capture any/all output to stdout or stderr, and not actually output it to stdout + or stderr. Yields and object with a get_captured_output() method to get the output captured thus far, + and another uncaptured_print() method to actually print the given output to stdout, even though output + to stdout is being captured. Can be useful, for example, in creating command-line scripts which invoke + code which outputs a lot of info, warning, error, etc to stdout or stderr, and we want to suprress that + output; but with the yielded uncaptured_print() method output specific to the script can actually be + output (to stdout); and/or can also optionally output any/all captured output, e.g. for debugging or + troubleshooting purposes. Disable this capture, without having to restructure your code WRT the usage + of the with-clause with this context manager, pass False as an argument to this context manager. + """ + + original_stdout = _real_stdout + original_stderr = _real_stderr + captured_output = io.StringIO() + + def set_original_output() -> None: + sys.stdout = original_stdout + sys.stderr = original_stderr + + def set_captured_output() -> None: + if capture: + sys.stdout = captured_output + sys.stderr = captured_output + + def uncaptured_print(*args, **kwargs) -> None: + set_original_output() + print(*args, **kwargs) + set_captured_output() + + def uncaptured_input(message: str) -> str: + set_original_output() + value = input(message) + set_captured_output() + return value + + def get_captured_output() -> Optional[str]: + return captured_output.getvalue() if capture else None + + try: + set_captured_output() + Result = namedtuple("Result", ["get_captured_output", "uncaptured_print", "uncaptured_input"]) + yield Result(get_captured_output, uncaptured_print, uncaptured_input) + finally: + set_original_output() + + +@contextmanager +def uncaptured_output(): + original_stdout = sys.stdout + original_stderr = sys.stderr + sys.stdout = _real_stdout + sys.stderr = _real_stderr + try: + yield + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr