8000 fix ASF / botocore CBOR decoding (#6494) · localstack/localstack@750d387 · GitHub
[go: up one dir, main page]
Skip to content

Commit 750d387

Browse files
alexrashedalexrashed
authored
fix ASF / botocore CBOR decoding (#6494)
1 parent 4967399 commit 750d387

File tree

3 files changed

+105
-3
lines changed

3 files changed

+105
-3
lines changed

‎localstack/aws/client.py

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
"""Utils to process AWS requests as a client."""
22
import io
33
import logging
4+
from datetime import datetime
45
from typing import Dict, Iterable, Optional
56

67
from botocore.model import OperationModel
7-
from botocore.parsers import ResponseParser
8-
from botocore.parsers import create_parser as create_response_parser
8+
from botocore.parsers import ResponseParser, ResponseParserFactory
99
from werkzeug import Response
1010

1111
from localstack.aws.api import CommonServiceException, ServiceException, ServiceResponse
12+
from localstack.runtime import hooks
13+
from localstack.utils.patch import patch
1214

1315
LOG = logging.getLogger(__name__)
1416

@@ -85,6 +87,38 @@ def _add_modeled_error_fields(
8587
parsed_response.update(modeled_parse)
8688

8789

90+
def _cbor_timestamp_parser(value):
91+
return datetime.fromtimestamp(value / 1000)
92+
93+
94+
def _cbor_blob_parser(value):
95+
return bytes(value)
96+
97+
98+
@hooks.on_infra_start()
99+
def _patch_botocore_json_parser():
100+
from botocore.parsers import BaseJSONParser
101+
102+
@patch(BaseJSONParser._parse_body_as_json)
103+
def _parse_body_as_json(fn, self, body_contents):
104+
"""
105+
botocore does not support CBOR encoded response parsing. Since we use the botocore parsers
106+
to parse responses from external backends (like kinesalite), we need to patch botocore to
107+
try CBOR decoding in case the JSON decoding fails.
108+
"""
109+
try:
110+
return fn(self, body_contents)
111+
except UnicodeDecodeError as json_exception:
112+
import cbor2
113+
114+
try:
115+
LOG.debug("botocore failed decoding JSON. Trying to decode as CBOR.")
116+
return cbor2.loads(body_contents)
117+
except Exception as cbor_exception:
118+
LOG.debug("CBOR fallback decoding failed.")
119+
raise cbor_exception from json_exception
120+
121+
88122
def parse_response(
89123
operation: OperationModel, response: Response, include_response_metadata: bool = True
90124
) -> ServiceResponse:
@@ -117,7 +151,14 @@ def parse_response(
117151
else:
118152
response_dict["body"] = response.data
119153

120-
parser = create_response_parser(operation.service_model.protocol)
154+
factory = ResponseParserFactory()
155+
if response.content_type and response.content_type.startswith("application/x-amz-cbor"):
156+
# botocore cannot handle CBOR encoded responses (because it never sends them), we need to modify the parser
157+
factory.set_parser_defaults(
158+
timestamp_parser=_cbor_timestamp_parser, blob_parser=_cbor_blob_parser
159+
)
160+
161+
parser = factory.create_parser(operation.service_model.protocol)
121162
parsed_response = parser.parse(response_dict, operation.output_shape)
122163

123164
if response.status_code >= 301:

‎localstack/aws/protocol/parser.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,15 @@ def _parse_boolean(
890890
) -> bool:
891891
return super()._noop_parser(request, shape, node, uri_params)
892892

893+
def _parse_blob(
894+
self, request: HttpRequest, shape: Shape, node: bool, uri_params: Mapping[str, Any] = None
895+
) -> bytes:
896+
if isinstance(node, bytes) and request.mimetype.startswith("application/x-amz-cbor"):
897+
# CBOR does not base64 encode binary data
898+
return bytes(node)
899+
else:
900+
return super()._parse_blob(request, shape, node, uri_params)
901+
893902

894903
class JSONRequestParser(BaseJSONRequestParser):
895904
"""

‎tests/unit/aws/protocol/test_parser.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,58 @@ def test_json_parser_cognito_with_botocore():
553553
)
554554

555555

556+
def test_json_cbor_blob_parsing():
557+
serialized_request = {
558+
"url_path": "/",
559+
"query_string": "",
560+
"method": "POST",
561+
"headers": {
562+
"Host": "localhost:4566",
563+
"amz-sdk-invocation-id": "d77968c6-b536-155d-7228-d4dfe6372154",
564+
"amz-sdk-request": "attempt=1; max=3",
565+
"Content-Length": "103",
566+
"Content-Type": "application/x-amz-cbor-1.1",
567+
"X-Amz-Date": "20220721T081553Z",
568+
"X-Amz-Target": "Kinesis_20131202.PutRecord",
569+
"x-localstack-tgt-api": "kinesis",
570+
},
571+
"body": b"\xbfjStreamNamedtestdDataMhello, world!lPartitionKeylpartitionkey\xff",
572+
"url": "/",
573+
"context": {},
574+
}
575+
576+
prepare_request_dict(serialized_request, "")
577+
split_url = urlsplit(serialized_request. F6ED get("url"))
578+
path = split_url.path
579+
query_string = split_url.query
580+
581+
# Use our parser to parse the serialized body
582+
# Load the appropriate service
583+
service = load_service("kinesis")
584+
operation_model = service.operation_model("PutRecord")
585+
parser = create_parser(service)
586+
parsed_operation_model, parsed_request = parser.parse(
587+
HttpRequest(
588+
method=serialized_request.get("method") or "GET",
589+
path=unquote(path),
590+
query_string=to_str(query_string),
591+
headers=serialized_request.get("headers"),
592+
body=serialized_request["body"],
593+
raw_path=path,
594+
)
595+
)
596+
597+
# Check if the determined operation_model is correct
598+
assert parsed_operation_model == operation_model
599+
600+
assert "Data" in parsed_request
601+
assert parsed_request["Data"] == b"hello, world!"
602+
assert "StreamName" in parsed_request
603+
assert parsed_request["StreamName"] == "test"
604+
assert "PartitionKey" in parsed_request
605+
assert parsed_request["PartitionKey"] == "partitionkey"
606+
607+
556608
def test_restjson_parser_xray_with_botocore():
557609
_botocore_parser_integration_test(
558610
service="xray",

0 commit comments

Comments
 (0)
0