8000 make config object hashable and equality-checkable, add tests for cac… · codeperl/localstack@ab92c16 · GitHub
[go: up one dir, main page]

Skip to content

Commit ab92c16

Browse files
authored
make config object hashable and equality-checkable, add tests for caching (localstack#9134)
1 parent 858bbf2 commit ab92c16

File tree

2 files changed

+69
-4
lines changed

2 files changed

+69
-4
lines changed

localstack/aws/connect.py

Lines changed: 33 additions & 3 deletions
8000
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import re
1010
import threading
1111
from abc import ABC, abstractmethod
12-
from functools import cache, partial
12+
from functools import lru_cache, partial
1313
from typing import Any, Callable, Generic, Optional, TypedDict, TypeVar
1414

1515
from boto3.session import Session
@@ -59,6 +59,36 @@ def my_patch(fn, self, **kwargs):
5959
return fn(self, **patched_kwargs)
6060

6161

62+
# patch the botocore.Config object to be comparable and hashable.
63+
# this solution does not validates the hashable (https://docs.python.org/3/glossary.html#term-hashable) definition on python
64+
# It would do so only when someone accesses the internals of the Config option to change the dict directly.
65+
# Since this is not a proper way to use the config object (but via config.merge), this should be fine
66+
def make_hash(o):
67+
if isinstance(o, (set, tuple, list)):
68+
return tuple([make_hash(e) for e in o])
69+
70+
elif not isinstance(o, dict):
71+
return hash(o)
72+
73+
new_o = {}
74+
for k, v in o.items():
75+
new_o[k] = make_hash(v)
76+
77+
return hash(frozenset(sorted(new_o.items())))
78+
79+
80+
def config_equality_patch(self, other: object):
81+
return type(self) == type(other) and self._user_provided_options == other._user_provided_options
82+
83+
84+
def config_hash_patch(self):
85+
return make_hash(self._user_provided_options)
86+
87+
88+
Config.__eq__ = config_equality_patch
89+
Config.__hash__ = config_hash_patch
90+
91+
6292
def attribute_name_to_service_name(attribute_name):
6393
"""
6494
Converts a python-compatible attribute name to the boto service name
@@ -314,10 +344,10 @@ def _get_client_post_hook(self, client: BaseClient) -> BaseClient:
314344
"""
315345
return client
316346

317-
# TODO @cache here might result in a memory leak, as it keeps a reference to `self`
347+
# TODO @lru_cache here might result in a memory leak, as it keeps a reference to `self`
318348
# We might need an alternative caching decorator with a weak ref to `self`
319349
# Otherwise factories might never be garbage collected
320-
@cache
350+
@lru_cache(maxsize=256)
321351
def _get_client(
322352
self,
323353
service_name: str,

tests/unit/aws/test_connect.py

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import boto3
44
import botocore
55
import pytest
6+
from botocore.config import Config
67

78
from localstack.aws.api import RequestContext
89
from localstack.aws.chain import Handler, HandlerChain
@@ -248,10 +249,44 @@ def test_client_caching(self):
248249
# TODO does it really make sense to test the caching?
249250
# TODO pretty ugly way of accessing the internal client
250251
factory = InternalClientFactory()
251-
assert factory().s3._client == factory().s3._client
252+
assert factory().s3._client is factory().s3._client
252253
factory_2 = InternalClientFactory()
253254
assert factory().s3._client != factory_2().s3._client
254255

256+
def test_client_caching_with_config(self):
257+
"""Test client caching. Same factory for the same service should result in the same client.
258+
Different factories should result in different (identity wise) clients"""
259+
# This test might get flaky if some internal boto3 caching is introduced at some point
260+
config = Config(read_timeout=2, signature_version=botocore.UNSIGNED)
261+
second_config = Config(read_timeout=2, signature_version=botocore.UNSIGNED)
262+
third_config = Config(read_timeout=3, signature_version=botocore.UNSIGNED)
263+
factory = InternalClientFactory()
264+
client_1 = factory(config=config).s3._client
265+
client_2 = factory(config=config).s3._client
266+
client_3 = factory(config=second_config).s3._client
267+
client_4 = factory(config=third_config).s3._client
268+
assert client_1 is client_2
269+
assert client_2 is client_3
270+
assert client_3 is not client_4
271+
272+
def test_client_caching_with_merged_configs(self):
273+
"""Test client caching. Same factory for the same service should result in the same client.
274+
Different factories should result in different (identity wise) clients"""
275+
# This test might get flaky if some internal boto3 caching is introduced at some point
276+
config_1 = Config(read_timeout=2)
277+
config_2 = Config(signature_version=botocore.UNSIGNED)
278+
config_3 = config_1.merge(config_2)
279+
config_4 = config_1.merge(config_2)
280+
factory = InternalClientFactory()
281+
client_1 = factory(config=config_1).s3._client
282+
client_2 = factory(config=config_2).s3._client
283+
client_3 = factory(config=config_3).s3._client
284+
client_4 = factory(config=config_4).s3._client
285+
assert client_1 is not client_2
286+
assert client_2 is not client_3
287+
assert client_1 is not client_3
288+
assert client_3 is client_4
289+
255290
def test_internal_request_parameters(self, create_dummy_request_parameter_gateway):
256291
internal_dto = None
257292

0 commit comments

Comments
 (0)
0