8000 Remove the obsolete legacy internal boto client (#9343) · codeperl/localstack@8d0336a · GitHub
[go: up one dir, main page]
Skip to content

Commit 8d0336a

Browse files
viren-nadkarniviren-nadkarni
authored
Remove the obsolete legacy internal boto client (localstack#9343)
1 parent 07041f0 commit 8d0336a
Copy full SHA for 8d0336a

File tree

1 file changed

+6
-241
lines changed

1 file changed

+6
-241
lines changed

‎localstack/utils/aws/aws_stack.py

Lines changed: 6 additions & 241 deletions
< 179B td data-grid-cell-id="diff-c249821a753f15353e8eaf413c9109fee4c2cfcab9df57c6fbf2fb9022270039-359-189-2" data-line-anchor="diff-c249821a753f15353e8eaf413c9109fee4c2cfcab9df57c6fbf2fb9022270039L359" data-selected="false" role="gridcell" style="background-color:var(--diffBlob-deletionLine-bgColor, var(--diffBlob-deletion-bgColor-line));padding-right:24px" tabindex="-1" valign="top" class="focusable-grid-cell diff-text-cell left-side-diff-cell pt-4 border-right left-side">-
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,12 @@
1-
import functools
21
import json
32
import logging
43
import os
54
import re
65
import socket
7-
import threading
8-
import warnings
96
from functools import lru_cache
107
from typing import Dict, Optional, Union
118

129
import boto3
13-
import botocore
14-
import botocore.config
1510

1611
from localstack import config
1712
from localstack.aws.accounts import get_aws_access_key_id, get_aws_account_id
@@ -23,11 +18,8 @@
2318
HEADER_LOCALSTACK_ACCOUNT_ID,
2419
INTERNAL_AWS_ACCESS_KEY_ID,
2520
LOCALHOST,
26-
MAX_POOL_CONNECTIONS,
2721
REGION_LOCAL,
2822
S3_VIRTUAL_HOSTNAME,
29-
TEST_AWS_ACCESS_KEY_ID,
30-
TEST_AWS_SECRET_ACCESS_KEY,
3123
)
3224
from localstack.utils.strings import is_string, is_string_or_bytes, to_str
3325

@@ -37,21 +29,12 @@
3729
# cache local region
3830
LOCAL_REGION = None
3931

40-
# Use this flag to enable creation of a new session for each boto3 connection.
41-
CREATE_NEW_SESSION_PER_BOTO3_CONNECTION = False
42-
4332
# Used in AWS assume role function
4433
INITIAL_BOTO3_SESSION = None
4534

46-
# Boto clients cache
47-
BOTO_CLIENTS_CACHE = {}
48-
4935
# cached value used to determine the DNS status of the S3 hostname (whether it can be resolved properly)
5036
CACHE_S3_HOSTNAME_DNS_STATUS = None
5137

52-
# mutex used when creating boto clients (which isn't thread safe: https://github.com/boto/boto3/issues/801)
53-
BOTO_CLIENT_CREATE_LOCK = threading.RLock()
54-
5538

5639
@lru_cache()
5740
def get_valid_regions():
@@ -62,13 +45,16 @@ def get_valid_regions():
6245
return valid_regions
6346

6447

48+
# FIXME: AWS recommends use of SSM parameter store to determine per region availability
49+
# https://github.com/aws/aws-sdk/issues/206#issuecomment-1471354853
6550
def get_valid_regions_for_service(service_name):
6651
regions = list(boto3.Session().get_available_regions(service_name))
6752
regions.extend(boto3.Session().get_available_regions("cloudwatch", partition_name="aws-us-gov"))
6853
regions.extend(boto3.Session().get_available_regions("cloudwatch", partition_name="aws-cn"))
6954
return regions
7055

7156

57+
# TODO: Remove, this is super legacy functionality
7258
class Environment:
7359
def __init__(self, region=None, prefix=None):
7460
# target is the runtime environment to use, e.g.,
@@ -110,6 +96,7 @@ def __str__(self):
11096
PREDEFINED_ENVIRONMENTS = {ENV_DEV: Environment(region=REGION_LOCAL, prefix=ENV_DEV)}
11197

11298

99+
# TODO: Remove
113100
def get_environment(env=None, region_name=None):
114101
"""
115102
Return an Environment object based on the input arguments.
@@ -144,29 +131,8 @@ def is_local_env(env):
144131
return not env or env.region == REGION_LOCAL or env.prefix == ENV_DEV
145132

146133

147-
class Boto3Session(boto3.session.Session):
148-
"""Custom boto3 session that points to local endpoint URLs."""
149-
150-
def resource(self, service, *args, **kwargs):
151-
self._fix_endpoint(kwargs)
152-
return connect_to_resource(service, *args, **kwargs)
153-
154-
def client(self, service, *args, **kwargs):
155-
self._fix_endpoint(kwargs)
156-
return connect_to_service(service, *args, **kwargs)
157-
158-
def _fix_endpoint(self, kwargs):
159-
if "amazonaws.com" in kwargs.get("endpoint_url", ""):
160-
kwargs.pop("endpoint_url")
161-
162-
163-
def get_boto3_session(cache=True):
164-
if not cache or CREATE_NEW_SESSION_PER_BOTO3_CONNECTION:
165-
return boto3.session.Session()
166-
# return default session
167-
return boto3
168-
169-
134+
# NOTE: This method should not be used as it is not guaranteed to return the correct region
135+
# In the near future it will be deprecated and removed
170136
def get_region():
171137
# Note: leave import here to avoid import errors (e.g., "flask") for CLI commands
172138
from localstack.utils.aws.request_context import get_region_from_request_context
@@ -221,189 +187,6 @@ def get_local_service_url(service_name_or_port: Union[str, int]) -> str:
221187
return config.service_url(service_name)
222188

223189

224-
def connect_to_resource(
225-
service_name, env=None, region_name=None, endpoint_url=None, *args, **kwargs
226-
):
227-
"""
228-
Generic method to obtain an AWS service resource using boto3, based on environment, region, or custom endpoint_url.
229-
"""
230-
warnings.warn(
231-
"`connect_to_resource` is obsolete. Use `localstack.aws.connect`", DeprecationWarning
232-
)
233-
234-
return connect_to_service(
235-
service_name,
236-
client=False,
237-
env=env,
238-
region_name=region_name,
239-
endpoint_url=endpoint_url,
240-
*args,
241-
**kwargs,
242-
)
243-
244-
245-
def connect_to_resource_external(
246-
service_name,
247-
env=None,
248-
region_name=None,
249-
endpoint_url=None,
250-
config: botocore.config.Config = None,
251-
**kwargs,
252-
):
253-
"""
254-
Generic method to obtain an AWS service resource using boto3, based on environment, region, or custom endpoint_url.
255-
"""
256-
warnings.warn(
257-
"`connect_to_resource_external` is obsolete. Use `localstack.aws.connect`",
258-
DeprecationWarning,
259-
)
260-
261-
return create_external_boto_client(
262-
service_name,
263-
client=False,
264-
env=env,
265-
region_name=region_name,
266-
endpoint_url=endpoint_url,
267-
config=config,
268-
)
269-
270-
271-
def connect_to_service(
272-
service_name,
273-
client=True,
274-
env=None,
275-
region_name=None,
276-
endpoint_url=None,
277-
config: botocore.config.Config = None,
278-
verify=False,
279-
cache=True,
280-
internal=True,
281-
*args,
282-
**kwargs,
283-
):
284-
"""
285-
Generic method to obtain an AWS service client using boto3, based on environment, region, or custom endpoint_url.
286-
"""
287-
warnings.warn(
288-
"`connect_to_service` is obsolete. Use `localstack.aws.connect`",
289-
DeprecationWarning,
290-
)
291-
292-
# determine context and create cache key
293-
region_name = region_name or get_region()
294-
env = get_environment(env, region_name=region_name)
295-
region = env.region if env.region != REGION_LOCAL else region_name
296-
key_elements = [service_name, client, env, region, endpoint_url, config, internal, kwargs]
297-
cache_key = "/".join([str(k) for k in key_elements])
298-
299-
# check cache first (most calls will be served from cache)
300-
if cache and cache_key in BOTO_CLIENTS_CACHE:
301-
return BOTO_CLIENTS_CACHE[cache_key]
302-
303-
with BOTO_CLIENT_CREATE_LOCK:
304-
# check cache again within lock context to avoid race conditions
305-
if cache and cache_key in BOTO_CLIENTS_CACHE:
306-
return BOTO_CLIENTS_CACHE[cache_key]
307-
308-
# determine endpoint_url if it is not set explicitly
309-
if not endpoint_url:
310-
if is_local_env(env):
311-
endpoint_url = get_local_service_url(service_name)
312-
verify = False
313-
backend_env_name = "%s_BACKEND" % service_name.upper()
314-
backend_url = os.environ.get(backend_env_name, "").strip()
315-
if backend_url:
316-
endpoint_url = backend_url
317-
318-
# configure S3 path/host style addressing
319-
if service_name == "s3":
320-
if re.match(r"https?://localhost(:[0-9]+)?", endpoint_url):
321-
endpoint_url = endpoint_url.replace("://localhost", "://%s" % get_s3_hostname())
322-
323-
# create boto client or resource from potentially cached session
324-
boto_session = get_boto3_session(cache=cache)
325-
boto_config = config or botocore.client.Config()
326-
boto_factory = boto_session.client if client else boto_session.resource
327-
328-
# To, prevent error "Connection pool is full, discarding connection ...",
329-
# set the environment variable MAX_POOL_CONNECTIONS. Default is 150.
330-
boto_config.max_pool_connections = MAX_POOL_CONNECTIONS
331-
332-
new_client = boto_factory(
333-
service_name,
334-
region_name=region,
335-
endpoint_url=endpoint_url,
336-
verify=verify,
337-
config=boto_config,
338-
**kwargs,
339-
)
340-
341-
# We set a custom header in all internal calls which help LocalStack
342-
# identify requests as such
343-
if client and internal:
344-
345-
def _add_internal_header(account_id: str, request, **kwargs):
346-
request.headers.add_header(HEADER_LOCALSTACK_ACCOUNT_ID, account_id)
347-
348-
# The handler invocation happens in boto context leading to loss of account ID
349-
# Hence we build a partial here with the account ID baked-in.
350-
_handler = functools.partial(
351-
_add_internal_header, kwargs.get("aws_access_key_id", get_aws_account_id())
352-
)
353-
354-
event_system = new_client.meta.events
355-
event_system.register_first("before-sign.*.*", _handler)
356-
357-
if cache:
358-
BOTO_CLIENTS_CACHE[cache_key] = new_client
359
360-
return new_client
361-
362-
363-
def create_external_boto_client(
364-
service_name,
365-
client=True,
366-
env=None,
367-
region_name=None,
368-
endpoint_url=None,
369-
config: botocore.config.Config = None,
370-
verify=False,
371-
cache=True,
372-
aws_access_key_id=None,
373-
aws_secret_access_key=None,
374-
*args,
375-
**kwargs,
376-
):
377-
warnings.warn(
378-
"`create_external_boto_client` is obsolete. Use `localstack.aws.connect`",
379-
DeprecationWarning,
380-
)
381-
382-
# Currently we use the Access Key ID field to specify the AWS account ID; this will change when IAM matures.
383-
# It is important that the correct Account ID is included in the request as that will determine access to namespaced resources.
384-
if aws_access_key_id is None:
385-
aws_access_key_id = get_aws_account_id()
386-
387-
if aws_secret_access_key is None:
388-
aws_secret_access_key = TEST_AWS_SECRET_ACCESS_KEY
389-
390-
return connect_to_service(
391-
service_name,
392-
client,
393-
env,
394-
region_name,
395-
endpoint_url,
396-
config,
397-
verify,
398-
cache,
399-
internal=False,
400-
aws_access_key_id=aws_access_key_id,
401-
aws_secret_access_key=aws_secret_access_key,
402-
*args,
403-
**kwargs,
404-
)
405-
406-
407190
def get_s3_hostname():
408191
global CACHE_S3_HOSTNAME_DNS_STATUS
409192
if CACHE_S3_HOSTNAME_DNS_STATUS is None:
@@ -417,24 +200,6 @@ def get_s3_hostname():
417200
return LOCALHOST
418201

419202

420-
def generate_presigned_url(*args, **kwargs):
421-
warnings.warn(
422-
"`aws_stack.generate_presigned_url` is obsolete. Use the Boto client `generate_presigned_url` method",
423-
DeprecationWarning,
424-
)
425-
426-
endpoint_url = kwargs.pop("endpoint_url", None)
427-
s3_client = connect_to_service(
428-
"s3",
429-
endpoint_url=endpoint_url,
430-
cache=False,
431-
# Note: presigned URL needs to be created with (external) test credentials
432-
aws_access_key_id=TEST_AWS_ACCESS_KEY_ID,
433-
aws_secret_access_key=TEST_AWS_SECRET_ACCESS_KEY,
434-
)
435-
return s3_client.generate_presigned_url(*args, **kwargs)
436-
437-
438203
def set_default_region_in_headers(headers, service=None, region=None):
439204
# this should now be a no-op, as we support arbitrary regions and don't use a "default" region
440205
# TODO: remove this function once the legacy USE_SINGLE_REGION config is removed

0 commit comments

Comments
 (0)
0