8000 New AWS client by dfangl · Pull Request #7240 · localstack/localstack · GitHub
[go: up one dir, main page]

Skip to content

New AWS client #7240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 57 commits into from
Mar 13, 2023
Merged
Changes from 1 commit
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
add7928
add mob-programming based client prototype
dfangl Nov 24, 2022
f57429d
WIP
viren-nadkarni Jan 2, 2023
ab0c79f
Merge branch 'master' into aws-client
viren-nadkarni Jan 4, 2023
89e43f5
Fix imports
viren-nadkarni Jan 4, 2023
247340c
Merge branch 'master' into aws-client
viren-nadkarni Jan 5, 2023
0352fae
Updates
viren-nadkarni Jan 6, 2023
548ced3
Fallback to default internal credentials
viren-nadkarni Jan 12, 2023
f6f37fa
Proper loading of default credentials
viren-nadkarni Jan 12, 2023
f392242
Move to its own module
viren-nadkarni Jan 12, 2023
f8edc9c
Fix datetime
viren-nadkarni Jan 13, 2023
b53e068
Allow module to be used for external clients also
viren-nadkarni Jan 16, 2023
f8d8d8e
Use headers for internal call arg func
viren-nadkarni Jan 16, 2023
9f67ea9
Add tests
viren-nadkarni Jan 18, 2023
ff8e9a4
Remove dev comments
viren-nadkarni Jan 18, 2023
b551b2a
Default access keys
viren-nadkarni Jan 18, 2023
a4461c0
Fixes
viren-nadkarni Jan 18, 2023
161e0ae
Merge branch 'master' into aws-client
viren-nadkarni Jan 18, 2023
a457707
Use separate functions for internal and external use
viren-nadkarni Jan 18, 2023
c0a01e1
Enhancements
viren-nadkarni Jan 18, 2023
94a9170
Update tests
viren-nadkarni Jan 18, 2023
cc93146
Merge branch 'master' into aws-stack-dto
viren-nadkarni Jan 20, 2023
d616ff6
WIP
viren-nadkarni Jan 23, 2023
2b2c463
Remove assertion from prod code
viren-nadkarni Jan 30, 2023
b6cfdea
Revamp hook logic
viren-nadkarni Jan 30, 2023
e39fabf
Merge branch 'master' into aws-stack-dto
viren-nadkarni Jan 31, 2023
6819110
Merge branch 'aws-client' into aws-stack-dto
viren-nadkarni Jan 31, 2023
70bf9f4
Merge branch 'master' into aws-client
viren-nadkarni Jan 31, 2023
206f295
Fixes
viren-nadkarni Jan 31, 2023
cac43ed
Merge branch 'aws-client' into aws-client-dto
viren-nadkarni Jan 31, 2023
aeed818
Add new enricher
viren-nadkarni Jan 31, 2023
0150e36
Fixes
viren-nadkarni Jan 31, 2023
2dbebad
Merge branch 'aws-client' into aws-client-dto
viren-nadkarni Jan 31, 2023
d4c251f
Fixes
viren-nadkarni Jan 31, 2023
3597fe8
Merge branch 'aws-client' into aws-client-dto
viren-nadkarni Jan 31, 2023
cc7a47c
Override region from target ARN
viren-nadkarni Jan 31, 2023
30a4e7c
Minor touches
viren-nadkarni Jan 31, 2023
2e91751
Allow no region when it is overridden
viren-nadkarni Jan 31, 2023
ac93d4c
Override account ID along with region for internal calls with TargetArns
viren-nadkarni Jan 31, 2023
d45e6ce
Remove SourceService
viren-nadkarni Feb 1, 2023
8036dcf
Prevent fallback account ID for internal calls
viren-nadkarni Feb 1, 2023
806f321
Update tests
viren-nadkarni Feb 1, 2023
9f3f3a2
Update note
viren-nadkarni Feb 2, 2023
daee913
Merge branch 'master' into aws-client
viren-nadkarni Feb 9, 2023
0d9ca21
Rename to ClientFactory
viren-nadkarni Feb 9, 2023
afaef80
Use inheritance to specialise factories
viren-nadkarni Feb 9, 2023
8bdcb10
Remove internal call helper
viren-nadkarni Feb 9, 2023
cc693d5
Fix tests
viren-nadkarni Feb 13, 2023
10b7d86
Merge branch 'master' into aws-client
viren-nadkarni Feb 13, 2023
46d690e
Merge branch 'master' into aws-client
viren-nadkarni Feb 15, 2023
f790be7
Merge branch 'master' into aws-client
dfangl Feb 27, 2023
48cdd07
add some preliminary changes
dfangl Feb 27, 2023
5d49524
add typed interface, add some tests + some test scaffolds
dfangl Mar 9, 2023
4b6ff91
Merge branch 'master' into aws-client
dfangl Mar 12, 2023
0641ae3
add more tests, fix internal call detection
dfangl Mar 13, 2023
f10f983
Apply suggestions from code review
dfangl Mar 13, 2023
db858a3
fix imports, remove unnecessary tests
dfangl Mar 13, 2023
dc7c36f
fix nits
dfangl Mar 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Allow module to be used for external clients also
  • Loading branch information
viren-nadkarni committed Jan 16, 2023
commit b53e0684d6610c43fc046c349602ecb510f67648
56 changes: 46 additions & 10 deletions localstack/aws/connect.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""
Internal AWS client.
LocalStack client stack.

This module provides the interface to perform cross-service communication between
LocalStack providers.
Expand All @@ -20,6 +20,7 @@
from botocore.config import Config

from localstack import config
from localstack.aws.api import RequestContext
from localstack.constants import (
INTERNAL_AWS_ACCESS_KEY_ID,
INTERNAL_AWS_SECRET_ACCESS_KEY,
Expand Down Expand Up @@ -59,11 +60,13 @@ class LocalStackData(TypedDict):
"""Service principal where the call originates, eg. `ec2`"""

target_arn: str
"""ARN of the resource being targetted."""
"""ARN of the resource being targeted."""


def dump_dto(data: LocalStackData) -> str:
# TODO@viren: Improve minification using custom JSONEncoder that use shortened keys

# To produce a compact JSON representation of DTO, remove spaces from separators
return json.dumps(data, separators=(",", ":"))


Expand All @@ -78,7 +81,7 @@ def load_dto(data: str) -> LocalStackData:

class ConnectFactory:
"""
Factory to build the internal AWS client.
Factory to build the AWS client.
"""

def __init__(
Expand Down Expand Up @@ -111,12 +114,31 @@ def __init__(
self._session = Session()
self._config = Config(max_pool_connections=MAX_POOL_CONNECTIONS)

def get_region(self) -> str:
def get_partition_for_region(self, region_name: str) -> str:
"""
Return the AWS partition name for a given region, eg. `aws`, `aws-cn`, etc.
"""
return self._session.get_partition_for_region(region_name)

def get_session_region_name(self) -> str:
"""
Return AWS region as set in the Boto session.
"""
return self._session.region_name

def get_region_name(self) -> str:
"""
Return the AWS region name from following sources, in order of availability.
- LocalStack request context
- LocalStack default region
- Boto session
"""
return (
get_region_from_request_context() or config.DEFAULT_REGION or self._session.region_name
get_region_from_request_context()
or config.DEFAULT_REGION
or self.get_session_region_name()
)

# TODO@viren is this thread safe?
@cache
def get_client(
self,
Expand Down Expand Up @@ -155,6 +177,9 @@ def __call__(
"""
Build and return the client.

Presence of any attribute apart from `source_*` or `target_*` argument
indicates that this is a client meant for internal calls.

:param target_service: Service to build the client for, eg. `s3`
:param region_name: Name of the AWS region to be associated with the client
:param endpoint_url: Full endpoint URL to be used by the client.
Expand All @@ -176,12 +201,13 @@ def __call__(
localstack_data["source_service"] = source_service

if target_arn:
# Attention: region is overriden here
region_name = extract_region_from_arn(target_arn)
localstack_data["target_arn"] = target_arn

client = self.get_client(
service_name=target_service,
region_name=region_name or self.get_region(),
region_name=region_name or self.get_region_name(),
use_ssl=self._use_ssl,
verify=self._verify,
endpoint_url=endpoint_url or get_local_service_url(target_service),
Expand All @@ -195,13 +221,23 @@ def _handler(request: AWSPreparedRequest, **_):
data = localstack_data | LocalStackData(
current_time=datetime.now(timezone.utc).isoformat()
)

# Use a compact JSON representation of DTO
request.headers[LOCALSTACK_DATA_HEADER] = dump_dto(data)

client.meta.events.register("before-send.*.*", handler=_handler)
if len(localstack_data):
client.meta.events.register("before-send.*.*", handler=_handler)

return client


connect_to = ConnectFactory()

#
# Utilities
#


def is_internal_call(context: RequestContext) -> bool:
"""
Whether a given request is an internal LocalStack cross-service call.
"""
return LOCALSTACK_DATA_HEADER in context.request.headers
0