8000 Use new AWS client in SES by viren-nadkarni · Pull Request #7484 · localstack/localstack · GitHub
[go: up one dir, main page]

Skip to content

Use new AWS client in SES #7484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 27 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
add7928
add mob-programming based client prototype
dfangl Nov 24, 2022
f57429d
WIP
viren-nadkarni Jan 2, 2023
ab0c79f
Merge branch 'master' into aws-client
viren-nadkarni Jan 4, 2023
89e43f5
Fix imports
viren-nadkarni Jan 4, 2023
247340c
Merge branch 'master' into aws-client
viren-nadkarni Jan 5, 2023
0352fae
Updates
viren-nadkarni Jan 6, 2023
5a0ba6c
Add core descriptor
viren-nadkarni Jan 6, 2023
3f399c6
Add unit tests
viren-nadkarni Jan 9, 2023
e1a1d77
Add owner for stores codebase
viren-nadkarni Jan 9, 2023
5f1eeb2
Enable cross account access for SNS topics
viren-nadkarni Jan 9, 2023
0247739
Fixes
viren-nadkarni Jan 10, 2023
0b34651
Add tests
viren-nadkarni Jan 10, 2023
0803163
Merge branch 'master' into sns-cross-account-access
viren-nadkarni Jan 11, 2023
2e9d609
Merge branch 'master' into sns-cross-account-access
viren-nadkarni Jan 12, 2023
8ec3aa9
Remove duplicate assignment
viren-nadkarni Jan 12, 2023
53fa98b
Merge branch 'aws-client' into ses-sns-caa
viren-nadkarni Jan 12, 2023
548ced3
Fallback to default internal credentials
viren-nadkarni Jan 12, 2023
f6f37fa
Proper loading of default credentials
viren-nadkarni Jan 12, 2023
401fcff
Merge branch 'aws-client' into ses-sns-caa
viren-nadkarni Jan 12, 2023
f392242
Move to its own module
viren-nadkarni Jan 12, 2023
d5b113d
Merge branch 'aws-client' into ses-sns-caa
viren-nadkarni Jan 12, 2023
f8edc9c
Fix datetime
viren-nadkarni Jan 13, 2023
f1f78e0
Merge branch 'aws-client' into ses-sns-caa
viren-nadkarni Jan 13, 2023
3a2a669
WIP
viren-nadkarni Jan 13, 2023
b53e068
Allow module to be used for external clients also
viren-nadkarni Jan 16, 2023
169f3fe
Use new client at all places SNS was previously being used
viren-nadkarni Jan 16, 2023
fe192f5
Merge branch 'aws-client' into ses-sns-caa
viren-nadkarni Jan 16, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 1 addition & 218 deletions localstack/aws/client.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,17 @@
"""Utils to process AWS requests as a client."""
import dataclasses
import io
import json
import logging
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, TypedDict
from typing import Dict, Iterable, Optional

from boto3 import Session
from botocore.awsrequest import AWSPreparedRequest
from botocore.client import BaseClient
from botocore.config import Config as BotoConfig
from botocore.model import OperationModel
from botocore.parsers import ResponseParser, ResponseParserFactory
from werkzeug import Response

from localstack.aws.api import CommonServiceException, ServiceException, ServiceResponse
from localstack.constants import INTERNAL_AWS_ACCESS_KEY_ID, INTERNAL_AWS_SECRET_ACCESS_KEY
from localstack.runtime import hooks
from localstack.utils.aws.arns import extract_region_from_arn
from localstack.utils.aws.aws_stack import get_local_service_url
from localstack.utils.patch import patch

if TYPE_CHECKING:
from mypy_boto3_sqs import SQSClient

LOG = logging.getLogger(__name__)


Expand Down Expand Up @@ -232,208 +220,3 @@ def raise_service_exception(response: Response, parsed_response: Dict) -> None:
"""
if service_exception := parse_service_exception(response, parsed_response):
raise service_exception


#
# Internal AWS client
#

"""
The internal AWS client API provides the means to perform cross-service
communication within LocalStack.

Any additional information LocalStack might need for the purpose of policy
enforcement is sent as a data transfer object. This is a serialised dict object
sent in the request header.
"""

LOCALSTACK_DATA_HEADER = "x-localstack-data"
"""Request header which contains the data transfer object."""


class LocalStackData(TypedDict):
"""
LocalStack Data Transfer Object.
"""

source_arn: str
source_service: str # eg. 'ec2.amazonaws.com'


class Credentials(TypedDict):
"""
AWS credentials.
"""

aws_access_key_id: str
aws_secret_access_key: str
aws_session_token: str


@dataclasses.dataclass(frozen=True)
class ClientOptions:
"""This object holds configuration options for the internal AWS client."""

region_name: Optional[str] = None
"""Name of the AWS region to be associated with the client."""

endpoint_url: Optional[str] = None
"""Full endpoint URL to be used by the client."""

use_ssl: bool = True
"""Whether or not to use SSL."""

verify: bool = True
"""Whether or not to verify SSL certificates."""

aws_access_key_id: Optional[str] = None
"""Access key to use for the client."""

aws_secret_access_key: Optional[str] = None
"""Secret key to use for the client."""

aws_session_token: Optional[str] = None
"""Session token to use for the client"""

boto_config: Optional[BotoConfig] = dataclasses.field(default_factory=BotoConfig)
"""Boto client configuration for advanced use."""

localstack_data: dict[str, Any] = dataclasses.field(default_factory=LocalStackData)
"""LocalStack data transfer object."""


class ClientFactory:
"""
Factory to build the internal AWS client.
"""

# TODO migrate to immutable clientfactory instances
client_options: ClientOptions
session: Session

def __init__(self, client_options: ClientOptions = None):
self.client_options = client_options or ClientOptions()
self.session = Session()

def with_endpoint(self, endpoint: str) -> "ClientFactory":
"""
Set a custom endpoint.
"""
return ClientFactory(
client_options=dataclasses.replace(self.client_options, endpoint_url=endpoint)
)

def with_source_arn(self, arn: str) -> "ClientFactory":
"""
Indicate that the client is operating from a given resource.

This must be used in cross-service requests.
"""
return ClientFactory(
client_options=dataclasses.replace(
self.client_options,
localstack_data=self.client_options.localstack_data
| LocalStackData(source_arn=arn),
)
)

def with_target_arn(self, arn: str) -> "ClientFactory":
"""
Create the client to operate on a target resource.

This must be used in cross-service requests.
"""
region_name = extract_region_from_arn(arn)
return ClientFactory(
client_options=dataclasses.replace(self.client_options, region_name=region_name)
)

def with_source_service_principal(self, source_service: str) -> "ClientFactory":
"""
Set the source service principal.

This must be used in cross-service requests.
"""
return ClientFactory(
client_options=dataclasses.replace(
self.client_options,
localstack_data=self.client_options.localstack_data
| LocalStackData(source_service=f"{source_service}.amazonaws.com"),
)
)

def with_credentials(
self, aws_access_key_id: str, aws_secret_access_key: str
) -> "ClientFactory":
"""
Use custom AWS credentials.
"""
return ClientFactory(
client_options=dataclasses.replace(
self.client_options,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
)
)

def with_env_credentials(self) -> "ClientFactory":
"""
Use AWS credentials from the following locations:
- Environment variables
- Credentials file `~/.aws/credentials`
- Config file `~/.aws/config`
"""
credentials = self.session.get_credentials()
return self.with_credentials(credentials.access_key, credentials.secret_key)

def with_boto_config(self, config: BotoConfig) -> "ClientFactory":
"""
Use a custom BotoConfig.
"""
return ClientFactory(
client_options=dataclasses.replace(
self.client_options, boto_config=self.client_options.boto_config.merge(config)
)
)

def build(self, service: str) -> BaseClient:
"""
Finalise the client.
"""
aws_access_key_id = self.client_options.aws_access_key_id or INTERNAL_AWS_ACCESS_KEY_ID
aws_secret_access_key = (
self.client_options.aws_secret_access_key or INTERNAL_AWS_SECRET_ACCESS_KEY
)
endpoint_url = self.client_options.endpoint_url or get_local_service_url(service)

# TODO@viren: creating a boto client is very intensive. In old aws_stack, we cache clients based on
# [service_name, client, env, region, endpoint_url, config, internal, kwargs]
# Come up with an appropriate solution here
client = self.session.client(
service_name=service,
config=self.client_options.boto_config,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=endpoint_url,
)

def event_handler(request: AWSPreparedRequest, **_):
# Send a compact JSON representation as DTO
request.headers[LOCALSTACK_DATA_HEADER] = json.dumps(
self.client_options.localstack_data, separators=(",", ":")
)

client.meta.events.register("before-send.*.*", handler=event_handler)

return client

#
# Convenience helpers
#

def sqs(self) -> "SQSClient":
return self.build("sqs")


def aws_client():
return ClientFactory()
Loading
0