localstack · alexrashed · Mar 5, 2025 · Feb 27, 2025 · Mar 4, 2025
diff --git a/Dockerfile b/Dockerfile
@@ -147,6 +147,9 @@ RUN --mount=type=cache,target=/root/.cache \
 RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LOCALSTACK_CORE=${LOCALSTACK_BUILD_VERSION} \
     make entrypoints
 
+# Generate service catalog cache in static libs dir
+RUN . .venv/bin/activate && python3 -m localstack.aws.spec
+
 # Install packages which should be shipped by default
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/var/lib/localstack/cache \

diff --git a/localstack-core/localstack/aws/protocol/service_router.py b/localstack-core/localstack/aws/protocol/service_router.py
@@ -1,24 +1,18 @@
 import logging
-import os
 from typing import NamedTuple, Optional, Set
 
-import botocore
 from botocore.model import ServiceModel
 from werkzeug.exceptions import RequestEntityTooLarge
 from werkzeug.http import parse_dict_header
 
-from localstack import config
 from localstack.aws.spec import (
     ServiceCatalog,
     ServiceModelIdentifier,
-    build_service_index_cache,
-    load_service_index_cache,
+    get_service_catalog,
 )
-from localstack.constants import VERSION
 from localstack.http import Request
 from localstack.services.s3.utils import uses_host_addressing
 from localstack.services.sqs.utils import is_sqs_queue_url
-from localstack.utils.objects import singleton_factory
 from localstack.utils.strings import to_bytes
 from localstack.utils.urls import hostname_from_url
 
@@ -264,33 +258,6 @@ def legacy_rules(request: Request) -> Optional[ServiceModelIdentifier]:
         return ServiceModelIdentifier("s3")
 
 
-@singleton_factory
-def get_service_catalog() -> ServiceCatalog:
-    """Loads the ServiceCatalog (which contains all the service specs), and potentially re-uses a cached index."""
-    if not os.path.isdir(config.dirs.cache):
-        return ServiceCatalog()
-
-    try:
-        ls_ver = VERSION.replace(".", "_")
-        botocore_ver = botocore.__version__.replace(".", "_")
-        cache_file_name = f"service-catalog-{ls_ver}-{botocore_ver}.pickle"
-        cache_file = os.path.join(config.dirs.cache, cache_file_name)
-
-        if not os.path.exists(cache_file):
-            LOG.debug("building service catalog index cache file %s", cache_file)
-            index = build_service_index_cache(cache_file)
-        else:
-            LOG.debug("loading service catalog index cache file %s", cache_file)
-            index = load_service_index_cache(cache_file)
-
-        return ServiceCatalog(index)
-    except Exception:
-        LOG.exception(
-            "error while processing service catalog index cache, falling back to lazy-loaded index"
-        )
-        return ServiceCatalog()
-
-
 def resolve_conflicts(
     candidates: Set[ServiceModelIdentifier], request: Request
 ) -> ServiceModelIdentifier:

diff --git a/localstack-core/localstack/aws/spec.py b/localstack-core/localstack/aws/spec.py
@@ -2,15 +2,21 @@
 import json
 import logging
 import os
+import sys
 from collections import defaultdict
 from functools import cached_property, lru_cache
 from typing import Dict, Generator, List, Literal, NamedTuple, Optional, Tuple
 
+import botocore
 import jsonpatch
 from botocore.exceptions import UnknownServiceError
 from botocore.loaders import Loader, instance_cache
 from botocore.model import OperationModel, ServiceModel
 
+from localstack import config
+from localstack.constants import VERSION
+from localstack.utils.objects import singleton_factory
+
 LOG = logging.getLogger(__name__)
 
 ServiceName = str
@@ -265,35 +271,35 @@ def build_service_index_cache(file_path: str) -> ServiceCatalogIndex:
     """
     Creates a new ServiceCatalogIndex and stores it into the given file_path.
 
-    :param file_path: the path to pickle to
+    :param file_path: the path to store the file to
     :return: the created ServiceCatalogIndex
     """
     return save_service_index_cache(LazyServiceCatalogIndex(), file_path)
 
 
 def load_service_index_cache(file: str) -> ServiceCatalogIndex:
     """
-    Loads from the given file the pickled ServiceCatalogIndex.
+    Loads from the given file the stored ServiceCatalogIndex.
 
     :param file: the file to load from
     :return: the loaded ServiceCatalogIndex
     """
-    import pickle
+    import dill
 
     with open(file, "rb") as fd:
-        return pickle.load(fd)
+        return dill.load(fd)
 
 
 def save_service_index_cache(index: LazyServiceCatalogIndex, file_path: str) -> ServiceCatalogIndex:
     """
-    Creates from the given LazyServiceCatalogIndex a ``ServiceCatalogIndex`, pickles its contents into the given file,
+    Creates from the given LazyServiceCatalogIndex a ``ServiceCatalogIndex`, stores its contents into the given file,
     and then returns the newly created index.
 
     :param index: the LazyServiceCatalogIndex to store the index from.
-    :param file_path: the path to pickle to
+    :param file_path: the path to store the binary index cache file to
     :return: the created ServiceCatalogIndex
     """
-    import pickle
+    import dill
 
     cache = ServiceCatalogIndex(
         service_names=index.service_names,
@@ -303,5 +309,61 @@ def save_service_index_cache(index: LazyServiceCatalogIndex, file_path: str) ->
         target_prefix_index=index.target_prefix_index,
     )
     with open(file_path, "wb") as fd:
-        pickle.dump(cache, fd)
+        # use dill (instead of plain pickle) to avoid issues when serializing the pickle from __main__
+        dill.dump(cache, fd)
     return cache
+
+
+def _get_catalog_filename():
+    ls_ver = VERSION.replace(".", "_")
+    botocore_ver = botocore.__version__.replace(".", "_")
+    return f"service-catalog-{ls_ver}-{botocore_ver}.dill"
+
+
+@singleton_factory
+def get_service_catalog() -> ServiceCatalog:
+    """Loads the ServiceCatalog (which contains all the service specs), and potentially re-uses a cached index."""
+
+    try:
+        catalog_file_name = _get_catalog_filename()
+        static_catalog_file = os.path.join(config.dirs.static_libs, catalog_file_name)
+
+        # try to load or load/build/save the service catalog index from the static libs
+        index = None
+        if os.path.exists(static_catalog_file):
+            # load the service catalog from the static libs dir / built at build time
+            LOG.debug("loading service catalog index cache file %s", static_catalog_file)
+            index = load_service_index_cache(static_catalog_file)
+        elif os.path.isdir(config.dirs.cache):
+            cache_catalog_file = os.path.join(config.dirs.cache, catalog_file_name)
+            if os.path.exists(cache_catalog_file):
+                LOG.debug("loading service catalog index cache file %s", cache_catalog_file)
+                index = load_service_index_cache(cache_catalog_file)
+            else:
+                LOG.debug("building service catalog index cache file %s", cache_catalog_file)
+                index = build_service_index_cache(cache_catalog_file)
+        return ServiceCatalog(index)
+    except Exception:
+        LOG.exception(
+            "error while processing service catalog index cache, falling back to lazy-loaded index"
+        )
+        return ServiceCatalog()
+
+
+def main():
+    catalog_file_name = _get_catalog_filename()
+    static_catalog_file = os.path.join(config.dirs.static_libs, catalog_file_name)
+
+    if os.path.exists(static_catalog_file):
+        LOG.error(
+            "service catalog index cache file (%s) already there. aborting!", static_catalog_file
+        )
+        return 1
+
+    # load the service catalog from the static libs dir / built at build time
+    LOG.debug("building service catalog index cache file %s", static_catalog_file)
+    build_service_index_cache(static_catalog_file)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/localstack-core/localstack/services/apigateway/next_gen/execute_api/integrations/aws.py b/localstack-core/localstack/services/apigateway/next_gen/execute_api/integrations/aws.py
@@ -17,7 +17,7 @@
     connect_to,
     dump_dto,
 )
-from localstack.aws.protocol.service_router import get_service_catalog
+from localstack.aws.spec import get_service_catalog
 from localstack.constants import APPLICATION_JSON, INTERNAL_AWS_ACCESS_KEY_ID
 from localstack.utils.aws.arns import extract_region_from_arn
 from localstack.utils.aws.client_types import ServicePrincipal

diff --git a/localstack-core/localstack/services/s3/cors.py b/localstack-core/localstack/services/s3/cors.py
@@ -18,7 +18,7 @@
 # TODO: refactor those to expose the needed methods
 from localstack.aws.handlers.cors import CorsEnforcer, CorsResponseEnricher
 from localstack.aws.protocol.op_router import RestServiceOperationRouter
-from localstack.aws.protocol.service_router import get_service_catalog
+from localstack.aws.spec import get_service_catalog
 from localstack.config import S3_VIRTUAL_HOSTNAME
 from localstack.http import Request, Response
 from localstack.services.s3.utils import S3_VIRTUAL_HOSTNAME_REGEX

diff --git a/localstack-core/localstack/services/s3/presigned_url.py b/localstack-core/localstack/services/s3/presigned_url.py
@@ -33,7 +33,7 @@
 )
 from localstack.aws.chain import HandlerChain
 from localstack.aws.protocol.op_router import RestServiceOperationRouter
-from localstack.aws.protocol.service_router import get_service_catalog
+from localstack.aws.spec import get_service_catalog
 from localstack.http import Request, Response
 from localstack.http.request import get_raw_path
 from localstack.services.s3.constants import (

diff --git a/...ions/asl/component/state/state_execution/state_task/service/state_task_service_aws_sdk.py b/...ions/asl/component/state/state_execution/state_task/service/state_task_service_aws_sdk.py
@@ -4,7 +4,7 @@
 from botocore.exceptions import ClientError, UnknownServiceError
 
 from localstack.aws.api.stepfunctions import HistoryEventType, TaskFailedEventDetails
-from localstack.aws.protocol.service_router import get_service_catalog
+from localstack.aws.spec import get_service_catalog
 from localstack.services.stepfunctions.asl.component.common.error_name.error_name import ErrorName
 from localstack.services.stepfunctions.asl.component.common.error_name.failure_event import (
     FailureEvent,

diff --git a/tests/unit/aws/test_service_router.py b/tests/unit/aws/test_service_router.py
@@ -7,7 +7,8 @@
 from botocore.config import Config
 from botocore.model import OperationModel, ServiceModel, Shape, StructureShape
 
-from localstack.aws.protocol.service_router import determine_aws_service_model, get_service_catalog
+from localstack.aws.protocol.service_router import determine_aws_service_model
+from localstack.aws.spec import get_service_catalog
 from localstack.http import Request
 from localstack.utils.run import to_str