localstack · giograno · Sep 9, 2024 · Sep 2, 2024 · Sep 3, 2024 · Sep 3, 2024
@@ -336,6 +336,7 @@ jobs:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DEBUG: 1
           DISABLE_BOTO_RETRIES: 1
+          OPENAPI_VALIDATE_RESPONSE: 1
           DNS_ADDRESS: 0
           LAMBDA_EXECUTOR: "local"
           LOCALSTACK_API_KEY: "test"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,3 +20,9 @@ repos:
     rev: v1.2.1
     hooks:
       - id: check-pinned-deps-for-needed-upgrade
+
+  - repo: https://github.com/python-openapi/openapi-spec-validator
+    rev: 0.7.1
+    hooks:
+      - id: openapi-spec-validator
+        files: .*openapi.*\.(json|yaml|yml)
diff --git a/Makefile b/Makefile
@@ -119,13 +119,13 @@ test-coverage: test	  ## Run automated tests and create coverage report
 lint:              		  ## Run code linter to check code style, check if formatter would make changes and check if dependency pins need to be updated
 	($(VENV_RUN); python -m ruff check --output-format=full . && python -m ruff format --check .)
 	$(VENV_RUN); pre-commit run check-pinned-deps-for-needed-upgrade --files pyproject.toml # run pre-commit hook manually here to ensure that this check runs in CI as well
-
+	$(VENV_RUN); openapi-spec-validator localstack-core/localstack/openapi.yaml
 
 lint-modified:     		  ## Run code linter to check code style, check if formatter would make changes on modified files, and check if dependency pins need to be updated because of modified files
 	($(VENV_RUN); python -m ruff check --output-format=full `git diff --diff-filter=d --name-only HEAD | grep '\.py$$' | xargs` && python -m ruff format --check `git diff --diff-filter=d --name-only HEAD | grep '\.py$$' | xargs`)
 	$(VENV_RUN); pre-commit run check-pinned-deps-for-needed-upgrade --files $(git diff master --name-only) # run pre-commit hook manually here to ensure that this check runs in CI as well
 
-check-aws-markers:     		  ## Lightweight check to ensure all AWS tests have proper compatibilty markers set
+check-aws-markers:
10000
     		  ## Lightweight check to ensure all AWS tests have proper compatibility markers set
 	($(VENV_RUN); python -m pytest --co tests/aws/)
 
 format:            		  ## Run ruff to format the whole codebase

@@ -36,7 +36,8 @@ def __init__(self, service_manager: ServiceManager = None) -> None:
                 handlers.parse_service_name,  # enforce_cors and content_decoder depend on the service name
                 handlers.enforce_cors,
                 handlers.content_decoder,
-                handlers.serve_localstack_resources,  # try to serve internal resources in /_localstack first
+                handlers.validate_request_schema,  # validate request schema for public LS endpoints
+                handlers.serve_localstack_resources,  # try to serve endpoints in /_localstack
                 handlers.serve_edge_router_rules,
                 # start aws handler chain
                 handlers.parse_pre_signed_url_request,
@@ -67,6 +68,7 @@ def __init__(self, service_manager: ServiceManager = None) -> None:
         # response post-processing
         self.response_handlers.extend(
             [
+                handlers.validate_response_schema,  # validate response schema for public LS endpoints
                 handlers.modify_service_response,
                 handlers.parse_service_response,
                 handlers.run_custom_response_handlers,

@@ -14,6 +14,7 @@
     presigned_url,
     region,
     service,
+    validation,
 )
 
 handle_runtime_shutdown = internal.RuntimeShutdownHandler()
@@ -28,6 +29,8 @@
 add_region_from_header = region.RegionContextEnricher()
 rewrite_region = region.RegionRewriter()
 add_internal_request_params = internal_requests.InternalRequestParamsEnricher()
+validate_request_schema = validation.OpenAPIRequestValidator()
+validate_response_schema = validation.OpenAPIResponseValidator()
 log_exception = logging.ExceptionLogger()
 log_response = logging.ResponseLogger()
 count_service_request = analytics.ServiceRequestCounter()

@@ -1,4 +1,4 @@
-"""Handlers for fallback logic, e.g., populating empty requests or defauling with default exceptions."""
+"""Handlers for fallback logic, e.g., populating empty requests or defaulting with default exceptions."""
 
 import logging
 

@@ -0,0 +1,88 @@
+"""
+Handlers for validating request and response schema against OpenAPI specs.
+"""
+
+import logging
+import os
+from pathlib import Path
+
+from openapi_core import OpenAPI
+from openapi_core.contrib.werkzeug import WerkzeugOpenAPIRequest, WerkzeugOpenAPIResponse
+from openapi_core.exceptions import OpenAPIError
+from openapi_core.validation.request.exceptions import (
+    RequestValidationError,
+)
+from openapi_core.validation.response.exceptions import ResponseValidationError
+
+from localstack import config
+from localstack.aws.api import RequestContext
+from localstack.aws.chain import Handler, HandlerChain
+from localstack.constants import INTERNAL_RESOURCE_PATH
+from localstack.http import Response
+
+LOG = logging.getLogger(__name__)
+
+
+# TODO: replace with from importlib.resources.files when https://github.com/python/importlib_resources/issues/311 is
+#   resolved. Import from a namespace package is broken when installing in editable mode.
+oas_path = os.path.join(os.path.dirname(__file__), "..", "..", "openapi.yaml")
+
+
+class OpenAPIValidator(Handler):
+    openapi: "OpenAPI"
+
+    def __init__(self) -> None:
+        path = Path(oas_path)
+        assert path.exists()
+        self.openapi = OpenAPI.from_path(path)
+
+
+class OpenAPIRequestValidator(OpenAPIValidator):
+    """
+    Validates the requests to the LocalStack public endpoints (the ones with a _localstack or _aws prefix) against
+    a OpenAPI specification.
+    """
+
+    def __call__(self, chain: HandlerChain, context: RequestContext, response: Response):
+        if not config.OPENAPI_VALIDATE_REQUEST:
+            return
+
+        path = context.request.path
+
+        if path.startswith(f"{INTERNAL_RESOURCE_PATH}/") or path.startswith("/_aws/"):
+            try:
+                self.openapi.validate_request(WerkzeugOpenAPIRequest(context.request))
+            except RequestValidationError as e:
+                # Note: in this handler we only check validation errors, e.g., wrong body, missing required in the body.
+                response.status_code = 400
+                response.set_json({"error": "Bad Request", "message": str(e)})
+                chain.stop()
+            except OpenAPIError:
+                # Other errors can be raised when validating a request against the OpenAPI specification.
+                #   The most common are: ServerNotFound, OperationNotFound, or PathNotFound.
+                #   We explicitly do not check any other error but RequestValidationError ones.
+                #   We shallow the exception to avoid excessive logging (e.g., a lot of ServerNotFound), as the only
+                #   purpose of this handler is to check for request validation errors.
+                pass
+
+
+class OpenAPIResponseValidator(OpenAPIValidator):
+    def __call__(self, chain: HandlerChain, context: RequestContext, response: Response):
+        # The use of this flag is intended for test only. Eventual errors are due to LocalStack implementation and not
+        #   to improper user usage of the endpoints.
+        if not config.OPENAPI_VALIDATE_RESPONSE:
+            return
+
+        path = context.request.path
+
+        if path.startswith(f"{INTERNAL_RESOURCE_PATH}/") or path.startswith("/_aws/"):
+            try:
+                self.openapi.validate_response(
+                    WerkzeugOpenAPIRequest(context.request),
+                    WerkzeugOpenAPIResponse(response),
+                )
+            except ResponseValidationError as exc:
+                LOG.error("Response validation failed for %s: $s", path, exc)
+                response.status_code = 500
+                response.set_json({"error": exc.__class__.__name__, "message": str(exc)})
+                chain.stop()
@@ -1259,6 +1259,8 @@ def use_custom_dns():
     "LS_LOG",
     "MAIN_CONTAINER_NAME",
     "MAIN_DOCKER_NETWORK",
+    "OPENAPI_VALIDATE_REQUEST",
+    "OPENAPI_VALIDATE_RESPONSE",
     "OPENSEARCH_ENDPOINT_STRATEGY",
     "OUTBOUND_HTTP_PROXY",
     "OUTBOUND_HTTPS_PROXY",

diff --git a/localstack-core/localstack/dev/run/configurators.py b/localstack-core/localstack/dev/run/configurators.py
@@ -107,7 +107,7 @@ def __call__(self, cfg: ContainerConfiguration):
 
 class SourceVolumeMountConfigurator:
     """
-    Mounts source code of localstack, localsack_ext, and moto into the container. It does this by assuming
+    Mounts source code of localstack, localstack_ext, and moto into the container. It does this by assuming
     that there is a "workspace" directory in which the source repositories are checked out into.
     Depending on whether we want to start the pro container, the source paths for localstack are different.
     """