8000 move service catalog loading to spec, implement static gen and load · localstack/localstack@fbcedbe · GitHub
[go: up one dir, main page]
Skip to content

Commit fbcedbe

Browse files
alexrashedalexrashed
committed
move service catalog loading to spec, implement static gen and load
1 parent 8cf2f63 commit fbcedbe

File tree

9 files changed

+71
-39
lines changed

9 files changed

+71
-39
lines changed

‎Dockerfile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ RUN --mount=type=cache,target=/root/.cache \
147147
RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LOCALSTACK_CORE=${LOCALSTACK_BUILD_VERSION} \
148148
make entrypoints
149149

150+
# Generate service catalog cache in static libs dir
151+
RUN . .venv/bin/activate && python3 -m localstack.aws.spec
152+
150153
# Install packages which should be shipped by default
151154
RUN --mount=type=cache,target=/root/.cache \
152155
--mount=type=cache,target=/var/lib/localstack/cache \

‎localstack-core/localstack/__init__.py

Whitespace-only changes.

‎localstack-core/localstack/aws/protocol/service_router.py

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,18 @@
11
import logging
2-
import os
32
from typing import NamedTuple, Optional, Set
43

5-
import botocore
64
from botocore.model import ServiceModel
75
from werkzeug.exceptions import RequestEntityTooLarge
86
from werkzeug.http import parse_dict_header
97

10-
from localstack import config
118
from localstack.aws.spec import (
129
ServiceCatalog,
1310
ServiceModelIdentifier,
14-
build_service_index_cache,
15-
load_service_index_cache,
11+
get_service_catalog,
1612
)
17-
from localstack.constants import VERSION
1813
from localstack.http import Request
1914
from localstack.services.s3.utils import uses_host_addressing
2015
from localstack.services.sqs.utils import is_sqs_queue_url
21-
from localstack.utils.objects import singleton_factory
2216
from localstack.utils.strings import to_bytes
2317
from localstack.utils.urls import hostname_from_url
2418

@@ -264,33 +258,6 @@ def legacy_rules(request: Request) -> Optional[ServiceModelIdentifier]:
264258
return ServiceModelIdentifier("s3")
265259

266260

267-
@singleton_factory
268-
def get_service_catalog() -> ServiceCatalog:
269-
"""Loads the ServiceCatalog (which contains all the service specs), and potentially re-uses a cached index."""
270-
if not os.path.isdir(config.dirs.cache):
271-
return ServiceCatalog()
272-
273-
try:
274-
ls_ver = VERSION.replace(".", "_")
275-
botocore_ver = botocore.__version__.replace(".", "_")
276-
cache_file_name = f"service-catalog-{ls_ver}-{botocore_ver}.pickle"
277-
cache_file = os.path.join(config.dirs.cache, cache_file_name)
278-
279-
if not os.path.exists(cache_file):
280-
LOG.debug("building service catalog index cache file %s", cache_file)
281-
index = build_service_index_cache(cache_file)
282-
else:
283-
LOG.debug("loading service catalog index cache file %s", cache_fi 685C le)
284-
index = load_service_index_cache(cache_file)
285-
286-
return ServiceCatalog(index)
287-
except Exception:
288-
LOG.exception(
289-
"error while processing service catalog index cache, falling back to lazy-loaded index"
290-
)
291-
return ServiceCatalog()
292-
293-
294261
def resolve_conflicts(
295262
candidates: Set[ServiceModelIdentifier], request: Request
296263
) -> ServiceModelIdentifier:

‎localstack-core/localstack/aws/spec.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,21 @@
22
import json
33
import logging
44
import os
5+
import sys
56
from collections import defaultdict
67
from functools import cached_property, lru_cache
78
from typing import Dict, Generator, List, Literal, NamedTuple, Optional, Tuple
89

10+
import botocore
911
import jsonpatch
1012
from botocore.exceptions import UnknownServiceError
1113
from botocore.loaders import Loader, instance_cache
1214
from botocore.model import OperationModel, ServiceModel
1315

16+
from localstack import config
17+
from localstack.constants import VERSION
18+
from localstack.utils.objects import singleton_factory
19+
1420
LOG = logging.getLogger(__name__)
1521

1622
ServiceName = str
@@ -305,3 +311,58 @@ def save_service_index_cache(index: LazyServiceCatalogIndex, file_path: str) ->
305311
with open(file_path, "wb") as fd:
306312
pickle.dump(cache, fd)
307313
return cache
314+
315+
316+
def _get_catalog_filename():
317+
ls_ver = VERSION.replace(".", "_")
318+
botocore_ver = botocore.__version__.replace(".", "_")
319+
return f"service-catalog-{ls_ver}-{botocore_ver}.pickle"
320+
321+
322+
@singleton_factory
323+
def get_service_catalog() -> ServiceCatalog:
324+
"""Loads the ServiceCatalog (which contains all the service specs), and potentially re-uses a cached index."""
325+
326+
try:
327+
catalog_file_name = _get_catalog_filename()
328+
static_catalog_file = os.path.join(config.dirs.static_libs, catalog_file_name)
329+
330+
# try to load or load/build/save the service catalog index from the static libs
331+
index = None
332+
if os.path.exists(static_catalog_file):
333+
# load the service catalog from the static libs dir / built at build time
334+
LOG.debug("loading service catalog index cache file %s", static_catalog_file)
335+
index = load_service_index_cache(static_catalog_file)
336+
elif os.path.isdir(config.dirs.cache):
337+
cache_catalog_file = os.path.join(config.dirs.cache, catalog_file_name)
338+
if os.path.exists(cache_catalog_file):
339+
LOG.debug("loading service catalog index cache file %s", cache_catalog_file)
340+
index = load_service_index_cache(cache_catalog_file)
341+
else:
342+
LOG.debug("building service catalog index cache file %s", cache_catalog_file)
343+
index = build_service_index_cache(cache_catalog_file)
344+
return ServiceCatalog(index)
345+
except Exception:
346+
LOG.exception(
347+
"error while processing service catalog index cache, falling back to lazy-loaded index"
348+
)
349+
return ServiceCatalog()
350+
351+
352+
def main():
353+
catalog_file_name = _get_catalog_filename()
354+
static_catalog_file = os.path.join(config.dirs.static_libs, catalog_file_name)
355+
356+
if os.path.exists(static_catalog_file):
357+
LOG.error(
358+
"service catalog index cache file (%s) already there. aborting!", static_catalog_file
359+
)
360+
return 1
361+
362+
# load the service catalog from the static libs dir / built at build time
363+
LOG.debug("building service catalog index cache file %s", static_catalog_file)
364+
build_service_index_cache(static_catalog_file)
365+
366+
367+
if __name__ == "__main__":
368+
sys.exit(main())

‎localstack-core/localstack/services/apigateway/next_gen/execute_api/integrations/aws.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
connect_to,
1818
dump_dto,
1919
)
20-
from localstack.aws.protocol.service_router import get_service_catalog
20+
from localstack.aws.spec import get_service_catalog
2121
from localstack.constants import APPLICATION_JSON, INTERNAL_AWS_ACCESS_KEY_ID
2222
from localstack.utils.aws.arns import extract_region_from_arn
2323
from localstack.utils.aws.client_types import ServicePrincipal

‎localstack-core/localstack/services/s3/cors.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# TODO: refactor those to expose the needed methods
1919
from localstack.aws.handlers.cors import CorsEnforcer, CorsResponseEnricher
2020
from localstack.aws.protocol.op_router import RestServiceOperationRouter
21-
from localstack.aws.protocol.service_router import get_service_catalog
21+
from localstack.aws.spec import get_service_catalog
2222
from localstack.config import S3_VIRTUAL_HOSTNAME
2323
from localstack.http import Request, Response
2424
from localstack.services.s3.utils import S3_VIRTUAL_HOSTNAME_REGEX

‎localstack-core/localstack/services/s3/presigned_url.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
)
3434
from localstack.aws.chain import HandlerChain
3535
from localstack.aws.protocol.op_router import RestServiceOperationRouter
36-
from localstack.aws.protocol.service_router import get_service_catalog
36+
from localstack.aws.spec import get_service_catalog
3737
from localstack.http import Request, Response
3838
from localstack.http.request import get_raw_path
3939
from localstack.services.s3.constants import (

‎localstack-core/localstack/services/stepfunctions/asl/component/state/state_execution/state_task/service/state_task_service_aws_sdk.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from botocore.exceptions import ClientError, UnknownServiceError
55

66
from localstack.aws.api.stepfunctions import HistoryEventType, TaskFailedEventDetails
7-
from localstack.aws.protocol.service_router import get_service_catalog
7+
from localstack.aws.spec import get_service_catalog
88
from localstack.services.stepfunctions.asl.component.common.error_name.error_name import ErrorName
99
from localstack.services.stepfunctions.asl.component.common.error_name.failure_event import (
1010
FailureEvent,

‎tests/unit/aws/test_service_router.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
from botocore.config import Config
88
from botocore.model import OperationModel, ServiceModel, Shape, StructureShape
99

10-
from localstack.aws.protocol.service_router import determine_aws_service_model, get_service_catalog
10+
from localstack.aws.protocol.service_router import determine_aws_service_model
11+
from localstack.aws.spec import get_service_catalog
1112
from localstack.http import Request
1213
from localstack.utils.run import to_str
1314

0 commit comments

Comments
 (0)
0