From c379ff8a56bb8fe1aba5f9005884a83afc9619e6 Mon Sep 17 00:00:00 2001 From: Abhinav Vedmala Date: Wed, 26 Feb 2025 16:57:16 -0500 Subject: [PATCH 1/3] Update Step Functions Parent ID Generation (#622) Updates the parent ID generation using upstream Step Functions context object to be able to handle retries Old behavior would result in the same parent ID for all retries, this new behavior will use the `retry_count` when it's non-zero. This way we don't break backwards compatibility with old layers that don't use this value in the common case of `redrive_count:0` --- src/trace/context/extractor.spec.ts | 4 +-- .../context/extractors/step-function.spec.ts | 2 +- src/trace/step-function-service.spec.ts | 11 ++++--- src/trace/step-function-service.ts | 30 +++++++++++++------ 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/trace/context/extractor.spec.ts b/src/trace/context/extractor.spec.ts index 49729efd..cb47b6f6 100644 --- a/src/trace/context/extractor.spec.ts +++ b/src/trace/context/extractor.spec.ts @@ -690,7 +690,7 @@ describe("TraceContextExtractor", () => { expect(traceContext).not.toBeNull(); expect(traceContext?.toTraceId()).toBe("1139193989631387307"); - expect(traceContext?.toSpanId()).toBe("5892738536804826142"); + expect(traceContext?.toSpanId()).toBe("7747304477664363642"); expect(traceContext?.sampleMode()).toBe("1"); expect(traceContext?.source).toBe("event"); }); @@ -1092,7 +1092,7 @@ describe("TraceContextExtractor", () => { const sentMessage = sentSegment.toString(); expect(sentMessage).toEqual( - '{"format": "json", "version": 1}\n{"id":"11111","trace_id":"1-5e272390-8c398be037738dc042009320","parent_id":"94ae789b969f1cc5","name":"datadog-metadata","start_time":1487076708,"end_time":1487076708,"type":"subsegment","metadata":{"datadog":{"root_span_metadata":{"execution_id":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf","redrive_count":"0","state_entered_time":"2022-12-08T21:08:19.224Z","state_name":"step-one"}}}}', + '{"format": "json", "version": 1}\n{"id":"11111","trace_id":"1-5e272390-8c398be037738dc042009320","parent_id":"94ae789b969f1cc5","name":"datadog-metadata","start_time":1487076708,"end_time":1487076708,"type":"subsegment","metadata":{"datadog":{"root_span_metadata":{"execution_id":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf","redrive_count":"0","retry_count":"2","state_entered_time":"2022-12-08T21:08:19.224Z","state_name":"step-one"}}}}', ); }); diff --git a/src/trace/context/extractors/step-function.spec.ts b/src/trace/context/extractors/step-function.spec.ts index 609b5ffa..1465ab61 100644 --- a/src/trace/context/extractors/step-function.spec.ts +++ b/src/trace/context/extractors/step-function.spec.ts @@ -72,7 +72,7 @@ describe("StepFunctionEventTraceExtractor", () => { expect(traceContext).not.toBeNull(); expect(traceContext?.toTraceId()).toBe("435175499815315247"); - expect(traceContext?.toSpanId()).toBe("5063839446130725204"); + expect(traceContext?.toSpanId()).toBe("8782364156266188026"); expect(traceContext?.sampleMode()).toBe("1"); expect(traceContext?.source).toBe("event"); }); diff --git a/src/trace/step-function-service.spec.ts b/src/trace/step-function-service.spec.ts index 1eb6a710..a96ed389 100644 --- a/src/trace/step-function-service.spec.ts +++ b/src/trace/step-function-service.spec.ts @@ -150,6 +150,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", }); @@ -163,6 +164,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", root_execution_id: @@ -179,6 +181,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", trace_id: "10593586103637578129", @@ -203,7 +206,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("1139193989631387307"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -218,7 +221,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("8676990472248253142"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -233,7 +236,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("10593586103637578129"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -258,7 +261,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("1139193989631387307"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); diff --git a/src/trace/step-function-service.ts b/src/trace/step-function-service.ts index 29510bc9..fe5fe548 100644 --- a/src/trace/step-function-service.ts +++ b/src/trace/step-function-service.ts @@ -6,6 +6,7 @@ import { Sha256 } from "@aws-crypto/sha256-js"; interface NestedStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; root_execution_id: string; @@ -15,6 +16,7 @@ interface NestedStepFunctionContext { interface LambdaRootStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; trace_id: string; @@ -25,6 +27,7 @@ interface LambdaRootStepFunctionContext { interface LegacyStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; } @@ -91,13 +94,14 @@ export class StepFunctionContextService { // Extract the common context variables const stateMachineContext = this.extractStateMachineContext(event); if (stateMachineContext === null) return; - const { execution_id, redrive_count, state_entered_time, state_name } = stateMachineContext; + const { execution_id, redrive_count, retry_count, state_entered_time, state_name } = stateMachineContext; if (typeof event["serverless-version"] === "string" && event["serverless-version"] === "v1") { if (typeof event.RootExecutionId === "string") { this.context = { execution_id, redrive_count, + retry_count, state_entered_time, state_name, root_execution_id: event.RootExecutionId, @@ -107,6 +111,7 @@ export class StepFunctionContextService { this.context = { execution_id, redrive_count, + retry_count, state_entered_time, state_name, trace_id: event["x-datadog-trace-id"], @@ -115,7 +120,13 @@ export class StepFunctionContextService { } as LambdaRootStepFunctionContext; } } else { - this.context = { execution_id, redrive_count, state_entered_time, state_name } as LegacyStepFunctionContext; + this.context = { + execution_id, + redrive_count, + retry_count, + state_entered_time, + state_name, + } as LegacyStepFunctionContext; } } @@ -139,17 +150,16 @@ export class StepFunctionContextService { return null; } - const redrivePostfix = this.context.redrive_count === "0" ? "" : `#${this.context.redrive_count}`; + const countsSuffix = + this.context.retry_count !== "0" || this.context.redrive_count !== "0" + ? `#${this.context.retry_count}#${this.context.redrive_count}` + : ""; const parentId = this.deterministicSha256HashToBigIntString( - this.context.execution_id + - "#" + - this.context.state_name + - "#" + - this.context.state_entered_time + - redrivePostfix, + `${this.context.execution_id}#${this.context.state_name}#${this.context.state_entered_time}${countsSuffix}`, PARENT_ID, ); + const sampleMode = SampleMode.AUTO_KEEP; try { @@ -209,6 +219,7 @@ export class StepFunctionContextService { private extractStateMachineContext(event: any): { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; } | null { @@ -216,6 +227,7 @@ export class StepFunctionContextService { return { execution_id: event.Execution.Id, redrive_count: (event.Execution.RedriveCount ?? "0").toString(), + retry_count: (event.State.RetryCount ?? "0").toString(), state_entered_time: event.State.EnteredTime, state_name: event.State.Name, }; From a2adc5344c1795201debd4cc612be31bdf39f384 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Mon, 3 Mar 2025 11:04:12 -0500 Subject: [PATCH 2/3] Create bundle and publish to govcloud script (#627) --- .gitlab/input_files/build.yaml.tpl | 28 +++++++ .gitlab/scripts/publish_layers.sh | 14 ++-- scripts/publish_govcloud_layers.sh | 124 +++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 5 deletions(-) create mode 100755 scripts/publish_govcloud_layers.sh diff --git a/.gitlab/input_files/build.yaml.tpl b/.gitlab/input_files/build.yaml.tpl index 3775f396..737d746e 100644 --- a/.gitlab/input_files/build.yaml.tpl +++ b/.gitlab/input_files/build.yaml.tpl @@ -175,3 +175,31 @@ publish npm package: - *node-before-script script: - .gitlab/scripts/publish_npm.sh + +{{ range $environment := (ds "environments").environments }} + +{{ if eq $environment.name "prod" }}signed {{ end }}layer bundle: + stage: {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + tags: ["arch:amd64"] + rules: + - if: '"{{ $environment.name }}" =~ /^sandbox/' + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + dependencies: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID}/ + name: datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_node*.zip datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} +{{ end }} diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 604ca748..97a38f17 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -95,14 +95,18 @@ if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then else # Running on prod if [ -z "$CI_COMMIT_TAG" ]; then - printf "[Error] No CI_COMMIT_TAG found.\n" - printf "Exiting script...\n" - exit 1 + # this happens during manual govcloud releases. + if [ -z "$VERSION" ]; then + printf "[Error] No CI_COMMIT_TAG or VERSION found.\n" + printf "Exiting script...\n" + exit 1 + else + printf "Using provided VERSION: $VERSION\n" + fi else printf "Tag found in environment: $CI_COMMIT_TAG\n" + VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi - - VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi # Target layer version diff --git a/scripts/publish_govcloud_layers.sh b/scripts/publish_govcloud_layers.sh new file mode 100755 index 00000000..68e1725e --- /dev/null +++ b/scripts/publish_govcloud_layers.sh @@ -0,0 +1,124 @@ +#! /usr/bin/env bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2025 Datadog, Inc. +# +# USAGE: download the layer bundle from the build pipeline in gitlab. Use the +# Download button on the `layer bundle` job. This will be a zip file containing +# all of the required layers. Run this script as follows: +# +# ENVIRONMENT=[us1-staging-fed or us1-fed] [PIPELINE_LAYER_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud_layers.sh +# +# protip: you can drag the zip file from finder into your terminal to insert +# its path. + +set -e + +NODE_VERSIONS=("18.12" "20.9" "22.11") + +LAYER_PACKAGE=$1 + +if [ -z "$LAYER_PACKAGE" ]; then + printf "[ERROR]: layer package not provided\n" + exit 1 +fi + +PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip) +echo package name: $PACKAGE_NAME + +if [ -z "$ENVIRONMENT" ]; then + printf "[ERROR]: ENVIRONMENT not specified\n" + exit 1 +fi + +if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-staging-fed-power-user] +# sso_start_url=https://start.us-gov-home.awsapps.com/directory/d-9867188aeb +# sso_account_id=553727695824 +# sso_role_name=power-user +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="sandbox" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-(signed-)?bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +elif [ $ENVIRONMENT = "us1-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-fed-engineering] +# sso_start_url=https://start.us-gov-west-1.us-gov-home.awsapps.com/directory/d-98671fdc8b +# sso_account_id=002406178527 +# sso_role_name=engineering +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="prod" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-signed-bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +else + printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n" + exit 1 +fi + +# Clean and recreate the .layers directory +echo "Cleaning .layers directory..." +rm -rf .layers +mkdir -p .layers + +echo "Copying layer files to .layers directory..." +TEMP_DIR=$(mktemp -d) +unzip $LAYER_PACKAGE -d $TEMP_DIR +cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/ + + +AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --" + +echo "Checking that you have access to the GovCloud AWS account" +$AWS_VAULT_PREFIX aws sts get-caller-identity + + +AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +# Determine the target regions +if [ -z "$REGIONS" ]; then + echo "Region not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + echo "Region specified: $REGIONS" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then + echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS" + echo "" + echo "EXITING SCRIPT." + exit 1 + fi +fi + +for region in $REGIONS +do + echo "Starting publishing layers for region $region..." + + for NODE_VERSION in "${NODE_VERSIONS[@]}"; do + echo "Publishing Layer for Node ${NODE_VERSION} in region ${region}" + + # Set environment variables for the publish script + export REGION=$region + export NODE_VERSION=$NODE_VERSION + + # Run the publish script with AWS credentials + $AWS_VAULT_PREFIX .gitlab/scripts/publish_layers.sh + done +done + +echo "Done!" \ No newline at end of file From f7ee0d26a70668401e98459e611381249483b183 Mon Sep 17 00:00:00 2001 From: Joey Zhao <5253430+joeyzhao2018@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:47:06 -0500 Subject: [PATCH 3/3] v10.122.0 with ddtrace@5.40.0 (#629) * v10.122.0 with ddtrace@5.40.0 * yarn.lock update --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 5c2c5bce..224842da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "datadog-lambda-js", - "version": "10.121.0", + "version": "10.122.0", "description": "Lambda client library that supports hybrid tracing in node js", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -29,7 +29,7 @@ "@types/node": "^20.12.10", "@types/promise-retry": "^1.1.3", "@types/shimmer": "^1.0.1", - "dd-trace": "^5.37.1", + "dd-trace": "^5.40.0", "jest": "^27.0.1", "mock-fs": "4.14.0", "nock": "13.5.4", diff --git a/yarn.lock b/yarn.lock index 2629a331..5a83a388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2816,10 +2816,10 @@ dc-polyfill@^0.1.3, dc-polyfill@^0.1.4: resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.6.tgz#c2940fa68ffb24a7bf127cc6cfdd15b39f0e7f02" integrity sha512-UV33cugmCC49a5uWAApM+6Ev9ZdvIUMTrtCO9fj96TPGOQiea54oeO3tiEVdVeo3J9N2UdJEmbS4zOkkEA35uQ== -dd-trace@^5.37.1: - version "5.37.1" - resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.37.1.tgz#d92af8bc7819b3d12efc43b78ef8764632eff631" - integrity sha512-RiaP1N2jgt+XEw1+Wx7I3FVpP+lOIkLxq08Li27nZYtWE1oa2fN3TLFOsxW47BzaCTg5gWJfGdLsEqUR9Od/Ew== +dd-trace@^5.40.0: + version "5.40.0" + resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.40.0.tgz#42633550bb015a8cf752587f8829fa0c7ae0be15" + integrity sha512-/UYVCcgpZ9LnnUvIJcNfd1Hj51i8HhqLOn9PCj5gK3wJUn6MY/ie/5da2ZaFtoK2DKQ9OZmFBITLV3+KDl4pjA== dependencies: "@datadog/libdatadog" "^0.4.0" "@datadog/native-appsec" "8.4.0"