S3 localstack ize

.github/workflows/main-publish.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           python-version: 3.9
       - name: Install Python dependencies for publish
-        run: pip install requests toml
+        run: pip install boto3 requests toml
       - name: Publish
         env:
           PYPI_USER: ${{ secrets.PYPI_USER }}

CHANGELOG.rst

@@ -6,6 +6,14 @@ dcicutils
 Change Log
 ----------
 
+7.6.0
+=====
+* Added ``boto_monkey_patching`` module to use monkey patching to override the endpoint URLs for
+  for S3 or SQS boto3 client/resource creation using the LOCALSTACK_S3_URL or LOCALSTACK_SQS_URL
+  environment variables to specify that these services should use a locally running ersatz
+  instance of S3 or SQS via localstack.
+
+
 7.5.2
 =====
 

dcicutils/__init__.py

@@ -0,0 +1 @@
+from . import boto_monkey_patching  # noqa

dcicutils/boto_monkey_patching.py

@@ -0,0 +1,58 @@
+# Module to monkey patch the boto3 client and resource functions to use a custom endpoint-url.
+# Originally introduced June 2023 for overriding certain boto3 services (e.g. s3, sqs) to use the
+# localstack utility, which provides a way to run some AWS services locally, for testing purposes.
+# Currently only supported for S3 and SQS. To use this set the environment variables LOCALSTACK_S3_URL
+# and/or LOCALSTACK_SQS_URL to the localstack URL, for example, http://localhost:4566.
+# Reference: https://localstack.cloud
+
+import boto3
+import os
+from typing import Optional
+
+
+_boto_client_original = boto3.client
+_boto_resource_original = boto3.resource
+_boto_service_overrides_supported = [
+    {"service": "s3", "env": "LOCALSTACK_S3_URL"},
+    {"service": "sqs", "env": "LOCALSTACK_SQS_URL"}
+]
+# This will entirely disable this feature; for troubleshooting only.
+_boto_monkey_patching_disabled = False
+
+# For import only in test_boto_monkey_patching.
+_boto_monkey_patching_services = [item["service"] for item in _boto_service_overrides_supported]
+
+
+# For import only in test_boto_monkey_patching.
+def _boto_monkey_patching_endpoint_url_environ_name(service: str) -> Optional[str]:
+    for item in _boto_service_overrides_supported:
+        if item["service"] == service:
+            return item["env"]
+    return None
+
+
+def _setup_monkey_patching_kwargs(*args, **kwargs) -> dict:
+    if not _boto_monkey_patching_disabled:
+        endpoint_url = kwargs.get("endpoint_url")
+        if not endpoint_url:
+            for service_override in _boto_service_overrides_supported:
+                if service_override["service"] in args:
+                    endpoint_url = os.environ.get(service_override["env"])
+                    if endpoint_url:
+                        kwargs["endpoint_url"] = endpoint_url
+                    break
+    return kwargs
+
+
+def _monkey_patched_boto_client(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_client_original(*args, **kwargs)
+
+
+def _monkey_patched_boto_resource(*args, **kwargs):
+    kwargs = _setup_monkey_patching_kwargs(*args, **kwargs)
+    return _boto_resource_original(*args, **kwargs)
+
+
+boto3.client = _monkey_patched_boto_client
+boto3.resource = _monkey_patched_boto_resource

docs/source/dcicutils.rst

@@ -23,6 +23,13 @@ beanstalk_utils
    :members:
 
 
+boto_monkey_patching
+^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: dcicutils.boto_monkey_patching
+   :members:
+
+
 codebuild_utils
 ^^^^^^^^^^^^^^^
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "7.5.2"
+version = "7.6.0.1b5"  # TODO: To become 7.6.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

test/conftest.py

@@ -1,4 +1,11 @@
 import os
+
+# Disable any AWS endpoint-url environment variables overrides for testing.
+if "S3_URL" in os.environ:
+    os.environ.pop("S3_URL")
+if "SQS_URL" in os.environ:
+    os.environ.pop("SQS_URL")
+
 import pytest
 import requests
 

test/test_boto_monkey_patching.py

@@ -0,0 +1,69 @@
+import boto3
+import os
+from typing import Optional
+from dcicutils.misc_utils import override_environ
+
+# Reach into the implementation details of the boto_monkey_patching module
+# to get list of boto3 services for which we support monkey patching, and to
+# get the environment variable name used to do the associated endpoint-url override.
+from dcicutils.boto_monkey_patching import _boto_monkey_patching_services
+from dcicutils.boto_monkey_patching import _boto_monkey_patching_endpoint_url_environ_name
+
+
+_override_endpoint_url = "http://localhost:4566"
+
+
+def _is_default_aws_endpoint_url(endpoint_url: str, service: str):
+    return (endpoint_url == f"https://{service}.amazonaws.com" or
+            endpoint_url == f"https://{service}.{os.environ.get('AWS_DEFAULT_REGION')}.amazonaws.com")
+
+
+def _environ_overrides(service: str, endpoint_url: Optional[str]) -> dict:
+    return {
+        _boto_monkey_patching_endpoint_url_environ_name(service): endpoint_url,
+        "AWS_DEFAULT_REGION": "us-east-1"
+    }
+
+
+def _test_boto_monkey_patching_client_without_overriding(service: str):
+    with override_environ(**_environ_overrides(service, None)):
+        s3_client = boto3.client(service)
+        assert _is_default_aws_endpoint_url(s3_client.meta.endpoint_url, service)
+
+
+def _test_boto_monkey_patching_resource_without_overriding(service: str):
+    with override_environ(**_environ_overrides(service, None)):
+        s3_resource = boto3.resource(service)
+        assert _is_default_aws_endpoint_url(s3_resource.meta.client._endpoint.host, service)
+
+
+def _test_boto_monkey_patching_client_with_overriding(service: str):
+    with override_environ(**_environ_overrides(service, _override_endpoint_url)):
+        s3_client = boto3.client(service)
+        assert s3_client.meta.endpoint_url == _override_endpoint_url
+
+
+def _test_boto_monkey_patching_resource_with_overriding(service: str):
+    with override_environ(**_environ_overrides(service, _override_endpoint_url)):
+        s3_resource = boto3.resource(service)
+        assert s3_resource.meta.client._endpoint.host == _override_endpoint_url
+
+
+def test_boto_monkey_patching_client_without_overriding():
+    for service in _boto_monkey_patching_services:
+        _test_boto_monkey_patching_client_without_overriding(service)
+
+
+def test_boto_monkey_patching_resource_without_overriding():
+    for service in _boto_monkey_patching_services:
+        _test_boto_monkey_patching_resource_without_overriding(service)
+
+
+def test_boto_monkey_patching_client_with_overriding():
+    for service in _boto_monkey_patching_services:
+        _test_boto_monkey_patching_client_with_overriding(service)
+
+
+def test_boto_monkey_patching_resource_with_overriding():
+    for service in _boto_monkey_patching_services:
+        _test_boto_monkey_patching_resource_with_overriding(service)