From 7d16e8aca6d76834ea69123d5289b9d764cb5509 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Thu, 14 Nov 2024 12:53:18 -0500 Subject: [PATCH 1/3] chore: [SVLS-5973] fix span pointer flow control (#538) --- datadog_lambda/span_pointers.py | 61 +++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/datadog_lambda/span_pointers.py b/datadog_lambda/span_pointers.py index 86c9e60b..e111469e 100644 --- a/datadog_lambda/span_pointers.py +++ b/datadog_lambda/span_pointers.py @@ -2,6 +2,7 @@ import logging import os from typing import List +from typing import Optional from ddtrace._trace._span_pointer import _SpanPointerDirection from ddtrace._trace._span_pointer import _SpanPointerDescription @@ -30,7 +31,7 @@ def calculate_span_pointers( return _calculate_dynamodb_span_pointers_for_event(event) except Exception as e: - logger.warning( + logger.debug( "failed to calculate span pointers for event: %s", e, ) @@ -75,7 +76,7 @@ def _calculate_s3_span_pointers_for_object_created_s3_information( etag = s3_information["object"]["eTag"] except KeyError as e: - logger.warning( + logger.debug( "missing s3 information required to make a span pointer: %s", e, ) @@ -86,17 +87,31 @@ def _calculate_s3_span_pointers_for_object_created_s3_information( _aws_s3_object_span_pointer_description, ) - return [ - _aws_s3_object_span_pointer_description( + try: + span_pointer_description = _aws_s3_object_span_pointer_description( + operation="S3.LambdaEvent", pointer_direction=_SpanPointerDirection.UPSTREAM, bucket=bucket, key=key, etag=etag, ) - ] + except TypeError: + # The older version of this function did not have an operation + # parameter. + span_pointer_description = _aws_s3_object_span_pointer_description( + pointer_direction=_SpanPointerDirection.UPSTREAM, + bucket=bucket, + key=key, + etag=etag, + ) + + if span_pointer_description is None: + return [] + + return [span_pointer_description] except Exception as e: - logger.warning( + logger.debug( "failed to generate S3 span pointer: %s", e, ) @@ -120,10 +135,13 @@ def _calculate_dynamodb_span_pointers_for_event_record( ) -> List[_SpanPointerDescription]: try: table_name = _extract_table_name_from_dynamodb_stream_record(record) + if table_name is None: + return [] + primary_key = record["dynamodb"]["Keys"] except Exception as e: - logger.warning( + logger.debug( "missing DynamoDB information required to make a span pointer: %s", e, ) @@ -134,23 +152,36 @@ def _calculate_dynamodb_span_pointers_for_event_record( _aws_dynamodb_item_span_pointer_description, ) - return [ - _aws_dynamodb_item_span_pointer_description( + try: + span_pointer_description = _aws_dynamodb_item_span_pointer_description( + operation="DynamoDB.LambdaEvent", pointer_direction=_SpanPointerDirection.UPSTREAM, table_name=table_name, primary_key=primary_key, ) - ] + except TypeError: + # The older version of this function did not have an operation + # parameter. + span_pointer_description = _aws_dynamodb_item_span_pointer_description( + pointer_direction=_SpanPointerDirection.UPSTREAM, + table_name=table_name, + primary_key=primary_key, + ) + + if span_pointer_description is None: + return [] + + return [span_pointer_description] except Exception as e: - logger.warning( + logger.debug( "failed to generate DynamoDB span pointer: %s", e, ) return [] -def _extract_table_name_from_dynamodb_stream_record(record) -> str: +def _extract_table_name_from_dynamodb_stream_record(record) -> Optional[str]: # Example eventSourceARN: # arn:aws:dynamodb:us-east-2:123456789012:table/my-table/stream/2024-06-10T19:26:16.525 event_source_arn = record["eventSourceARN"] @@ -159,10 +190,12 @@ def _extract_table_name_from_dynamodb_stream_record(record) -> str: ":", maxsplit=5 ) if _arn != "arn" or _aws != "aws" or _dynamodb != "dynamodb": - raise ValueError(f"unexpected eventSourceARN format: {event_source_arn}") + logger.debug("unexpected eventSourceARN format: %s", event_source_arn) + return None [_table, table_name, _stream, _timestamp] = dynamodb_info.split("/") if _table != "table" or _stream != "stream": - raise ValueError(f"unexpected eventSourceARN format: {event_source_arn}") + logger.debug("unexpected eventSourceARN format: %s", event_source_arn) + return None return table_name From 57d1a818130d2b1bf347ae5517e072e2cc769baa Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Thu, 14 Nov 2024 12:56:58 -0500 Subject: [PATCH 2/3] chore: simplified GovCloud release script (#536) --- scripts/publish_prod.sh | 138 +++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 60 deletions(-) diff --git a/scripts/publish_prod.sh b/scripts/publish_prod.sh index f96c021d..f3d13653 100755 --- a/scripts/publish_prod.sh +++ b/scripts/publish_prod.sh @@ -4,6 +4,8 @@ set -e +read -p "Are we only doing the simplified GovCloud release? ONLY IF THE NORMAL RELEASE IS DONE AND YOU HAVE DOWNLOADED THE LAYERS (y/n)? " GOVCLOUD_ONLY + # Ensure on main, and pull the latest BRANCH=$(git rev-parse --abbrev-ref HEAD) if [ $BRANCH != "main" ]; then @@ -32,54 +34,65 @@ else fi # Ensure pypi registry access -read -p "Do you have access to PyPI (y/n)?" CONT -if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 +if [ $GOVCLOUD_ONLY == "y" ]; then + echo "Skipping PyPI check since this is a GovCloud-only release" + +else + read -p "Do you have access to PyPI (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 + fi fi CURRENT_VERSION=$(poetry version --short) LAYER_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2) -read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT -if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 -fi +if [ $GOVCLOUD_ONLY == "y" ]; then + echo "Skipping Libary Updates, code changes, layer builds and signing for GovCloud-only release" -echo "Answer 'n' if already done in a PR" -read -p "Update pyproject.toml version? (y/n)?" CONT -if [ "$CONT" != "y" ]; then - echo "Skipping updating package.json version" else - echo - echo "Replacing version in pyproject.toml and datadog_lambda/version.py" - echo - - poetry version ${NEW_VERSION} - echo "__version__ = \"${NEW_VERSION}\"" > datadog_lambda/version.py -fi - -echo -echo "Building layers..." -./scripts/build_layers.sh + read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 + fi + + echo "Answer 'n' if already done in a PR" + read -p "Update pyproject.toml version? (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Skipping updating package.json version" + else + echo + echo "Replacing version in pyproject.toml and datadog_lambda/version.py" + echo + + poetry version ${NEW_VERSION} + echo "__version__ = \"${NEW_VERSION}\"" > datadog_lambda/version.py + fi -echo -echo "Signing layers for commercial AWS regions" -aws-vault exec sso-prod-engineering -- ./scripts/sign_layers.sh prod - -echo "Answer 'n' if GitLab already did this" -read -p "Deploy layers to commercial AWS (y/n)?" CONT -if [ "$CONT" != "y" ]; then - echo "Skipping deployment to commercial AWS" -else - echo "Ensuring you have access to the production AWS account" - aws-vault exec sso-prod-engineering -- aws sts get-caller-identity + echo + echo "Building layers..." + ./scripts/build_layers.sh echo - echo "Publishing layers to commercial AWS regions" - VERSION=$LAYER_VERSION aws-vault exec sso-prod-engineering --no-session -- ./scripts/publish_layers.sh + echo "Signing layers for commercial AWS regions" + aws-vault exec sso-prod-engineering -- ./scripts/sign_layers.sh prod + + echo "Answer 'n' if GitLab already did this" + read -p "Deploy layers to commercial AWS (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Skipping deployment to commercial AWS" + else + echo "Ensuring you have access to the production AWS account" + aws-vault exec sso-prod-engineering -- aws sts get-caller-identity + + echo + echo "Publishing layers to commercial AWS regions" + VERSION=$LAYER_VERSION aws-vault exec sso-prod-engineering --no-session -- ./scripts/publish_layers.sh + fi fi + read -p "Deploy layers to GovCloud AWS (y/n)?" CONT if [ "$CONT" != "y" ]; then echo "Skipping deployment to GovCloud AWS" @@ -91,32 +104,37 @@ else VERSION=$LAYER_VERSION aws-vault exec sso-govcloud-us1-fed-engineering -- ./scripts/publish_layers.sh fi -echo "Answer 'n' if GitLab already did this" -read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT -if [ "$CONT" != "y" ]; then - echo "Skipping publishing to PyPI" -else - echo - echo "Publishing to https://pypi.org/project/datadog-lambda/" - ./scripts/pypi.sh -fi - +if [ $GOVCLOUD_ONLY == "y" ]; then + echo "Skipping PyPI check and Github Release since this is a GovCloud-only release" -echo "Answer 'n' if you already released in GitHub" -read -p "Do you want to bump the version in GitHub? (y/n)" CONT -if [ "$CONT" != "y" ]; then - echo "Skipping publishing updates to GitHub" else + echo "Answer 'n' if GitLab already did this" + read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Skipping publishing to PyPI" + else + echo + echo "Publishing to https://pypi.org/project/datadog-lambda/" + ./scripts/pypi.sh + fi + + + echo "Answer 'n' if you already released in GitHub" + read -p "Do you want to bump the version in GitHub? (y/n)" CONT + if [ "$CONT" != "y" ]; then + echo "Skipping publishing updates to GitHub" + else + echo + echo 'Publishing updates to github' + git commit pyproject.toml datadog_lambda/version.py -m "Bump version to ${NEW_VERSION}" + git push origin main + git tag "v$LAYER_VERSION" + git push origin "refs/tags/v$LAYER_VERSION" + fi + echo - echo 'Publishing updates to github' - git commit pyproject.toml datadog_lambda/version.py -m "Bump version to ${NEW_VERSION}" - git push origin main - git tag "v$LAYER_VERSION" - git push origin "refs/tags/v$LAYER_VERSION" + echo "Now create a new release with the tag v${LAYER_VERSION} created unless you have done this already" + echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION" fi - -echo -echo "Now create a new release with the tag v${LAYER_VERSION} created unless you have done this already" -echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION" # Open a PR to the documentation repo to automatically bump layer version VERSION=$LAYER_VERSION LAYER=datadog-lambda-python ./scripts/create_documentation_pr.sh From 49df8ee8321f191ce5215e30e820c096a5131377 Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Fri, 15 Nov 2024 09:36:22 -0500 Subject: [PATCH 3/3] chore: v6.102.0 release candidate (#539) --- datadog_lambda/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datadog_lambda/version.py b/datadog_lambda/version.py index 9d6a2f29..f8489c6d 100644 --- a/datadog_lambda/version.py +++ b/datadog_lambda/version.py @@ -1 +1 @@ -__version__ = "6.101.0" +__version__ = "6.102.0" diff --git a/pyproject.toml b/pyproject.toml index 7bb5df47..4acdb842 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datadog_lambda" -version = "6.101.0" +version = "6.102.0" description = "The Datadog AWS Lambda Library" authors = ["Datadog, Inc. "] license = "Apache-2.0"