diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..93e6be8034d --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,17 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version: 3, 3.9, 3.8, 3.7, 3.6 +ARG VARIANT="3.9" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} +ENV POETRY_VIRTUALENVS_IN_PROJECT=true + +# [Option] Install Node.js +ARG INSTALL_NODE="true" +ARG NODE_VERSION="lts/*" +RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +RUN pip3 install -U pip black poetry pre-commit + +# [Optional] Uncomment this section to install additional OS packages. +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends bash-completion diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..995ac6c4fba --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,47 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + "VARIANT": "3.9", + "INSTALL_NODE": "true", + "NODE_VERSION": "lts/*" + } + }, + "settings": { + "git.enableCommitSigning": true, + "editor.formatOnSave": true, + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.nosetestsEnabled": false, + "python.testing.pytestEnabled": true, + "python.formatting.provider": "black" + }, + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "littlefoxteam.vscode-python-test-adapter", + "ms-azuretools.vscode-docker", + "davidanson.vscode-markdownlint", + "bungcip.better-toml" + ], + "postCreateCommand": "make dev", + "remoteUser": "vscode" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c20c21747..a24786a6ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) fo ## [Unreleased] +## 1.20.1 - 2021-08-22 + +### Bug Fixes + +* **idempotency:** sorting keys before hashing ([#639](https://github.com/awslabs/aws-lambda-powertools-python/issues/639)) + +### Maintenance + +* markdown linter fixes ([#636](https://github.com/awslabs/aws-lambda-powertools-python/issues/636)) +* setup codespaces ([#637](https://github.com/awslabs/aws-lambda-powertools-python/issues/637)) +* **license:** add third party license ([#635](https://github.com/awslabs/aws-lambda-powertools-python/issues/635)) + ## 1.20.0 - 2021-08-21 ## Bug Fixes diff --git a/MANIFEST.in b/MANIFEST.in index f3155af7064..370ff1a55e0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE include README.md +include THIRD-PARTY-LICENSES recursive-exclude * __pycache__ recursive-exclude * *.py[co] diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 2f5dd512ac6..4901e9f9f75 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -223,7 +223,7 @@ def _generate_hash(self, data: Any) -> str: """ data = getattr(data, "raw_event", data) # could be a data class depending on decorator order - hashed_data = self.hash_function(json.dumps(data, cls=Encoder).encode()) + hashed_data = self.hash_function(json.dumps(data, cls=Encoder, sort_keys=True).encode()) return hashed_data.hexdigest() def _validate_payload(self, data: Dict[str, Any], data_record: DataRecord) -> None: @@ -310,7 +310,7 @@ def save_success(self, data: Dict[str, Any], result: dict) -> None: result: dict The response from function """ - response_data = json.dumps(result, cls=Encoder) + response_data = json.dumps(result, cls=Encoder, sort_keys=True) data_record = DataRecord( idempotency_key=self._get_hashed_idempotency_key(data=data), diff --git a/docs/core/event_handler/api_gateway.md b/docs/core/event_handler/api_gateway.md index 8e24ba9f5f3..7186e8412d1 100644 --- a/docs/core/event_handler/api_gateway.md +++ b/docs/core/event_handler/api_gateway.md @@ -435,7 +435,6 @@ Additionally, we provide pre-defined errors for the most popular ones such as HT return app.resolve(event, context) ``` - ### Custom Domain API Mappings When using Custom Domain API Mappings feature, you must use **`strip_prefixes`** param in the `ApiGatewayResolver` constructor. @@ -444,7 +443,6 @@ Scenario: You have a custom domain `api.mydomain.dev` and set an API Mapping `pa This will lead to a HTTP 404 despite having your Lambda configured correctly. See the example below on how to account for this change. - === "app.py" ```python hl_lines="7" @@ -459,7 +457,7 @@ This will lead to a HTTP 404 despite having your Lambda configured correctly. Se @app.get("/subscriptions/") @tracer.capture_method def get_subscription(subscription): - return {"subscription_id": subscription} + return {"subscription_id": subscription} @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) @tracer.capture_lambda_handler diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index c75f41b39ec..6cd487a2092 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -50,7 +50,6 @@ Same example as above, but using the `event_source` decorator if 'helloworld' in event.path and event.http_method == 'GET': do_something_with(event.body, user) ``` - **Autocomplete with self-documented properties and methods** ![Utilities Data Classes](../media/utilities_data_classes.png) @@ -93,9 +92,9 @@ Use **`APIGatewayAuthorizerRequestEvent`** for type `REQUEST` and **`APIGatewayA === "app_type_request.py" - This example uses the `APIGatewayAuthorizerResponse` to decline a given request if the user is not found. + This example uses the `APIGatewayAuthorizerResponse` to decline a given request if the user is not found. - When the user is found, it includes the user details in the request context that will be available to the back-end, and returns a full access policy for admin users. + When the user is found, it includes the user details in the request context that will be available to the back-end, and returns a full access policy for admin users. ```python hl_lines="2-5 26-31 36-37 40 44 46" from aws_lambda_powertools.utilities.data_classes import event_source @@ -185,7 +184,7 @@ See also [this blog post](https://aws.amazon.com/blogs/compute/introducing-iam-a === "app.py" - This example looks up user details via `x-token` header. It uses `APIGatewayAuthorizerResponseV2` to return a deny policy when user is not found or authorized. + This example looks up user details via `x-token` header. It uses `APIGatewayAuthorizerResponseV2` to return a deny policy when user is not found or authorized. ```python hl_lines="2-5 21 24" from aws_lambda_powertools.utilities.data_classes import event_source @@ -290,7 +289,7 @@ In this example extract the `requestId` as the `correlation_id` for logging, use def get_user_by_token(token: str): """Look a user by token""" - ... + ... @logger.inject_lambda_context(correlation_id_path=correlation_paths.APPSYNC_AUTHORIZER) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index d941946b681..495fe626d4f 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -40,7 +40,6 @@ Configuration | Value | Notes Partition key | `id` | TTL attribute name | `expiration` | This can only be configured after your table is created if you're using AWS Console - !!! tip "You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add your `function_name` in addition to the idempotency key as a hash key. @@ -127,13 +126,11 @@ Similar to [idempotent decorator](#idempotent-decorator), you can use `idempoten When using `idempotent_function`, you must tell us which keyword parameter in your function signature has the data we should use via **`data_keyword_argument`** - Such data must be JSON serializable. - - !!! warning "Make sure to call your decorated function using keyword arguments" === "app.py" - This example also demonstrates how you can integrate with [Batch utility](batch.md), so you can process each record in an idempotent manner. + This example also demonstrates how you can integrate with [Batch utility](batch.md), so you can process each record in an idempotent manner. ```python hl_lines="4 13 18 25" import uuid @@ -160,9 +157,9 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo @sqs_batch_processor(record_handler=record_handler) def lambda_handler(event, context): - # `data` parameter must be called as a keyword argument to work + # `data` parameter must be called as a keyword argument to work dummy("hello", "universe", data="test") - return {"statusCode": 200} + return {"statusCode": 200} ``` === "Example event" @@ -196,7 +193,6 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo } ``` - ### Choosing a payload subset for idempotency !!! tip "Dealing with always changing payloads" diff --git a/pyproject.toml b/pyproject.toml index 7b7b5a9c4ec..f8f34c7c33d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aws_lambda_powertools" -version = "1.20.0" +version = "1.20.1" description = "A suite of utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, batching, idempotency, feature flags, and more." authors = ["Amazon Web Services"] include = ["aws_lambda_powertools/py.typed"] diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index e613bb85e60..2c528cafc50 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -21,6 +21,10 @@ TABLE_NAME = "TEST_TABLE" +def serialize(data): + return json.dumps(data, sort_keys=True, cls=Encoder) + + @pytest.fixture(scope="module") def config() -> Config: return Config(region_name="us-east-1") @@ -62,12 +66,12 @@ def lambda_response(): @pytest.fixture(scope="module") def serialized_lambda_response(lambda_response): - return json.dumps(lambda_response, cls=Encoder) + return serialize(lambda_response) @pytest.fixture(scope="module") def deserialized_lambda_response(lambda_response): - return json.loads(json.dumps(lambda_response, cls=Encoder)) + return json.loads(serialize(lambda_response)) @pytest.fixture @@ -144,7 +148,7 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali def hashed_idempotency_key(lambda_apigw_event, default_jmespath, lambda_context): compiled_jmespath = jmespath.compile(default_jmespath) data = compiled_jmespath.search(lambda_apigw_event) - return "test-func#" + hashlib.md5(json.dumps(data).encode()).hexdigest() + return "test-func#" + hashlib.md5(serialize(data).encode()).hexdigest() @pytest.fixture @@ -152,12 +156,12 @@ def hashed_idempotency_key_with_envelope(lambda_apigw_event): event = extract_data_from_envelope( data=lambda_apigw_event, envelope=envelopes.API_GATEWAY_HTTP, jmespath_options={} ) - return "test-func#" + hashlib.md5(json.dumps(event).encode()).hexdigest() + return "test-func#" + hashlib.md5(serialize(event).encode()).hexdigest() @pytest.fixture def hashed_validation_key(lambda_apigw_event): - return hashlib.md5(json.dumps(lambda_apigw_event["requestContext"]).encode()).hexdigest() + return hashlib.md5(serialize(lambda_apigw_event["requestContext"]).encode()).hexdigest() @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 5505a7dc5c9..cb0d43ae6fa 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -21,6 +21,7 @@ from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent, idempotent_function from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer, DataRecord from aws_lambda_powertools.utilities.validation import envelopes, validator +from tests.functional.idempotency.conftest import serialize from tests.functional.utils import load_event TABLE_NAME = "TEST_TABLE" @@ -741,7 +742,7 @@ def test_default_no_raise_on_missing_idempotency_key( hashed_key = persistence_store._get_hashed_idempotency_key({}) # THEN return the hash of None - expected_value = "test-func#" + md5(json.dumps(None).encode()).hexdigest() + expected_value = "test-func#" + md5(serialize(None).encode()).hexdigest() assert expected_value == hashed_key @@ -785,7 +786,7 @@ def test_jmespath_with_powertools_json( expected_value = [sub_attr_value, key_attr_value] api_gateway_proxy_event = { "requestContext": {"authorizer": {"claims": {"sub": sub_attr_value}}}, - "body": json.dumps({"id": key_attr_value}), + "body": serialize({"id": key_attr_value}), } # WHEN calling _get_hashed_idempotency_key @@ -869,7 +870,7 @@ def _delete_record(self, data_record: DataRecord) -> None: def test_idempotent_lambda_event_source(lambda_context): # Scenario to validate that we can use the event_source decorator before or after the idempotent decorator mock_event = load_event("apiGatewayProxyV2Event.json") - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} # GIVEN an event_source decorator @@ -889,7 +890,7 @@ def lambda_handler(event, _): def test_idempotent_function(): # Scenario to validate we can use idempotent_function with any function mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -906,7 +907,7 @@ def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -921,7 +922,7 @@ def record_handler(arg_one, arg_two, record, is_record): def test_idempotent_function_invalid_data_kwarg(): mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} keyword_argument = "payload" @@ -938,7 +939,7 @@ def record_handler(record): def test_idempotent_function_arg_instead_of_kwarg(): mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} keyword_argument = "record" @@ -956,7 +957,7 @@ def record_handler(record): def test_idempotent_function_and_lambda_handler(lambda_context): # Scenario to validate we can use both idempotent_function and idempotent decorators mock_event = {"data": "value"} - persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(mock_event).encode()).hexdigest()) + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(serialize(mock_event).encode()).hexdigest()) expected_result = {"message": "Foo"} @idempotent_function(persistence_store=persistence_layer, data_keyword_argument="record") @@ -976,3 +977,20 @@ def lambda_handler(event, _): # THEN we expect the function and lambda handler to execute successfully assert fn_result == expected_result assert handler_result == expected_result + + +def test_idempotent_data_sorting(): + # Scenario to validate same data in different order hashes to the same idempotency key + data_one = {"data": "test message 1", "more_data": "more data 1"} + data_two = {"more_data": "more data 1", "data": "test message 1"} + + # Assertion will happen in MockPersistenceLayer + persistence_layer = MockPersistenceLayer("test-func#" + hashlib.md5(json.dumps(data_one).encode()).hexdigest()) + + # GIVEN + @idempotent_function(data_keyword_argument="payload", persistence_store=persistence_layer) + def dummy(payload): + return {"message": "hello"} + + # WHEN + dummy(payload=data_two)