diff --git a/.gitlab/input_files/build.yaml.tpl b/.gitlab/input_files/build.yaml.tpl index 3775f396..737d746e 100644 --- a/.gitlab/input_files/build.yaml.tpl +++ b/.gitlab/input_files/build.yaml.tpl @@ -175,3 +175,31 @@ publish npm package: - *node-before-script script: - .gitlab/scripts/publish_npm.sh + +{{ range $environment := (ds "environments").environments }} + +{{ if eq $environment.name "prod" }}signed {{ end }}layer bundle: + stage: {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + tags: ["arch:amd64"] + rules: + - if: '"{{ $environment.name }}" =~ /^sandbox/' + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + dependencies: + {{ range $runtime := (ds "runtimes").runtimes }} + - {{ if eq $environment.name "prod" }}sign{{ else }}build{{ end }} layer ({{ $runtime.name }}) + {{ end }} + artifacts: + expire_in: 1 day + paths: + - datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID}/ + name: datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + script: + - rm -rf datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - mkdir -p datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} + - cp .layers/datadog_lambda_node*.zip datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} +{{ end }} diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 604ca748..97a38f17 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -95,14 +95,18 @@ if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then else # Running on prod if [ -z "$CI_COMMIT_TAG" ]; then - printf "[Error] No CI_COMMIT_TAG found.\n" - printf "Exiting script...\n" - exit 1 + # this happens during manual govcloud releases. + if [ -z "$VERSION" ]; then + printf "[Error] No CI_COMMIT_TAG or VERSION found.\n" + printf "Exiting script...\n" + exit 1 + else + printf "Using provided VERSION: $VERSION\n" + fi else printf "Tag found in environment: $CI_COMMIT_TAG\n" + VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi - - VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi # Target layer version diff --git a/package.json b/package.json index 5c2c5bce..224842da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "datadog-lambda-js", - "version": "10.121.0", + "version": "10.122.0", "description": "Lambda client library that supports hybrid tracing in node js", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -29,7 +29,7 @@ "@types/node": "^20.12.10", "@types/promise-retry": "^1.1.3", "@types/shimmer": "^1.0.1", - "dd-trace": "^5.37.1", + "dd-trace": "^5.40.0", "jest": "^27.0.1", "mock-fs": "4.14.0", "nock": "13.5.4", diff --git a/scripts/publish_govcloud_layers.sh b/scripts/publish_govcloud_layers.sh new file mode 100755 index 00000000..68e1725e --- /dev/null +++ b/scripts/publish_govcloud_layers.sh @@ -0,0 +1,124 @@ +#! /usr/bin/env bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2025 Datadog, Inc. +# +# USAGE: download the layer bundle from the build pipeline in gitlab. Use the +# Download button on the `layer bundle` job. This will be a zip file containing +# all of the required layers. Run this script as follows: +# +# ENVIRONMENT=[us1-staging-fed or us1-fed] [PIPELINE_LAYER_SUFFIX=optional-layer-suffix] [REGIONS=us-gov-west-1] ./scripts/publish_govcloud_layers.sh +# +# protip: you can drag the zip file from finder into your terminal to insert +# its path. + +set -e + +NODE_VERSIONS=("18.12" "20.9" "22.11") + +LAYER_PACKAGE=$1 + +if [ -z "$LAYER_PACKAGE" ]; then + printf "[ERROR]: layer package not provided\n" + exit 1 +fi + +PACKAGE_NAME=$(basename "$LAYER_PACKAGE" .zip) +echo package name: $PACKAGE_NAME + +if [ -z "$ENVIRONMENT" ]; then + printf "[ERROR]: ENVIRONMENT not specified\n" + exit 1 +fi + +if [ "$ENVIRONMENT" = "us1-staging-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-staging-fed-power-user + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-staging-fed-power-user] +# sso_start_url=https://start.us-gov-home.awsapps.com/directory/d-9867188aeb +# sso_account_id=553727695824 +# sso_role_name=power-user +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="sandbox" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-(signed-)?bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +elif [ $ENVIRONMENT = "us1-fed" ]; then + AWS_VAULT_ROLE=sso-govcloud-us1-fed-engineering + +# this role looks like this in ~/.aws/config: +# [profile sso-govcloud-us1-fed-engineering] +# sso_start_url=https://start.us-gov-west-1.us-gov-home.awsapps.com/directory/d-98671fdc8b +# sso_account_id=002406178527 +# sso_role_name=engineering +# sso_region=us-gov-west-1 +# region=us-gov-west-1 + + export STAGE="prod" + if [[ ! "$PACKAGE_NAME" =~ ^datadog_lambda_js-signed-bundle-[0-9]+$ ]]; then + echo "[ERROR]: Unexpected package name: $PACKAGE_NAME" + exit 1 + fi + +else + printf "[ERROR]: ENVIRONMENT not supported, must be us1-staging-fed or us1-fed.\n" + exit 1 +fi + +# Clean and recreate the .layers directory +echo "Cleaning .layers directory..." +rm -rf .layers +mkdir -p .layers + +echo "Copying layer files to .layers directory..." +TEMP_DIR=$(mktemp -d) +unzip $LAYER_PACKAGE -d $TEMP_DIR +cp -v $TEMP_DIR/$PACKAGE_NAME/*.zip .layers/ + + +AWS_VAULT_PREFIX="aws-vault exec $AWS_VAULT_ROLE --" + +echo "Checking that you have access to the GovCloud AWS account" +$AWS_VAULT_PREFIX aws sts get-caller-identity + + +AVAILABLE_REGIONS=$($AWS_VAULT_PREFIX aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +# Determine the target regions +if [ -z "$REGIONS" ]; then + echo "Region not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + echo "Region specified: $REGIONS" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then + echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS" + echo "" + echo "EXITING SCRIPT." + exit 1 + fi +fi + +for region in $REGIONS +do + echo "Starting publishing layers for region $region..." + + for NODE_VERSION in "${NODE_VERSIONS[@]}"; do + echo "Publishing Layer for Node ${NODE_VERSION} in region ${region}" + + # Set environment variables for the publish script + export REGION=$region + export NODE_VERSION=$NODE_VERSION + + # Run the publish script with AWS credentials + $AWS_VAULT_PREFIX .gitlab/scripts/publish_layers.sh + done +done + +echo "Done!" \ No newline at end of file diff --git a/src/trace/context/extractor.spec.ts b/src/trace/context/extractor.spec.ts index 49729efd..cb47b6f6 100644 --- a/src/trace/context/extractor.spec.ts +++ b/src/trace/context/extractor.spec.ts @@ -690,7 +690,7 @@ describe("TraceContextExtractor", () => { expect(traceContext).not.toBeNull(); expect(traceContext?.toTraceId()).toBe("1139193989631387307"); - expect(traceContext?.toSpanId()).toBe("5892738536804826142"); + expect(traceContext?.toSpanId()).toBe("7747304477664363642"); expect(traceContext?.sampleMode()).toBe("1"); expect(traceContext?.source).toBe("event"); }); @@ -1092,7 +1092,7 @@ describe("TraceContextExtractor", () => { const sentMessage = sentSegment.toString(); expect(sentMessage).toEqual( - '{"format": "json", "version": 1}\n{"id":"11111","trace_id":"1-5e272390-8c398be037738dc042009320","parent_id":"94ae789b969f1cc5","name":"datadog-metadata","start_time":1487076708,"end_time":1487076708,"type":"subsegment","metadata":{"datadog":{"root_span_metadata":{"execution_id":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf","redrive_count":"0","state_entered_time":"2022-12-08T21:08:19.224Z","state_name":"step-one"}}}}', + '{"format": "json", "version": 1}\n{"id":"11111","trace_id":"1-5e272390-8c398be037738dc042009320","parent_id":"94ae789b969f1cc5","name":"datadog-metadata","start_time":1487076708,"end_time":1487076708,"type":"subsegment","metadata":{"datadog":{"root_span_metadata":{"execution_id":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf","redrive_count":"0","retry_count":"2","state_entered_time":"2022-12-08T21:08:19.224Z","state_name":"step-one"}}}}', ); }); diff --git a/src/trace/context/extractors/step-function.spec.ts b/src/trace/context/extractors/step-function.spec.ts index 609b5ffa..1465ab61 100644 --- a/src/trace/context/extractors/step-function.spec.ts +++ b/src/trace/context/extractors/step-function.spec.ts @@ -72,7 +72,7 @@ describe("StepFunctionEventTraceExtractor", () => { expect(traceContext).not.toBeNull(); expect(traceContext?.toTraceId()).toBe("435175499815315247"); - expect(traceContext?.toSpanId()).toBe("5063839446130725204"); + expect(traceContext?.toSpanId()).toBe("8782364156266188026"); expect(traceContext?.sampleMode()).toBe("1"); expect(traceContext?.source).toBe("event"); }); diff --git a/src/trace/step-function-service.spec.ts b/src/trace/step-function-service.spec.ts index 1eb6a710..a96ed389 100644 --- a/src/trace/step-function-service.spec.ts +++ b/src/trace/step-function-service.spec.ts @@ -150,6 +150,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", }); @@ -163,6 +164,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", root_execution_id: @@ -179,6 +181,7 @@ describe("StepFunctionContextService", () => { execution_id: "arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:85a9933e-9e11-83dc-6a61-b92367b6c3be:3f7ef5c7-c8b8-4c88-90a1-d54aa7e7e2bf", redrive_count: "0", + retry_count: "2", state_entered_time: "2022-12-08T21:08:19.224Z", state_name: "step-one", trace_id: "10593586103637578129", @@ -203,7 +206,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("1139193989631387307"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -218,7 +221,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("8676990472248253142"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -233,7 +236,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("10593586103637578129"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); @@ -258,7 +261,7 @@ describe("StepFunctionContextService", () => { expect(spanContext).not.toBeNull(); expect(spanContext?.toTraceId()).toBe("1139193989631387307"); - expect(spanContext?.toSpanId()).toBe("5892738536804826142"); + expect(spanContext?.toSpanId()).toBe("7747304477664363642"); expect(spanContext?.sampleMode()).toBe("1"); expect(spanContext?.source).toBe("event"); }); diff --git a/src/trace/step-function-service.ts b/src/trace/step-function-service.ts index 29510bc9..fe5fe548 100644 --- a/src/trace/step-function-service.ts +++ b/src/trace/step-function-service.ts @@ -6,6 +6,7 @@ import { Sha256 } from "@aws-crypto/sha256-js"; interface NestedStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; root_execution_id: string; @@ -15,6 +16,7 @@ interface NestedStepFunctionContext { interface LambdaRootStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; trace_id: string; @@ -25,6 +27,7 @@ interface LambdaRootStepFunctionContext { interface LegacyStepFunctionContext { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; } @@ -91,13 +94,14 @@ export class StepFunctionContextService { // Extract the common context variables const stateMachineContext = this.extractStateMachineContext(event); if (stateMachineContext === null) return; - const { execution_id, redrive_count, state_entered_time, state_name } = stateMachineContext; + const { execution_id, redrive_count, retry_count, state_entered_time, state_name } = stateMachineContext; if (typeof event["serverless-version"] === "string" && event["serverless-version"] === "v1") { if (typeof event.RootExecutionId === "string") { this.context = { execution_id, redrive_count, + retry_count, state_entered_time, state_name, root_execution_id: event.RootExecutionId, @@ -107,6 +111,7 @@ export class StepFunctionContextService { this.context = { execution_id, redrive_count, + retry_count, state_entered_time, state_name, trace_id: event["x-datadog-trace-id"], @@ -115,7 +120,13 @@ export class StepFunctionContextService { } as LambdaRootStepFunctionContext; } } else { - this.context = { execution_id, redrive_count, state_entered_time, state_name } as LegacyStepFunctionContext; + this.context = { + execution_id, + redrive_count, + retry_count, + state_entered_time, + state_name, + } as LegacyStepFunctionContext; } } @@ -139,17 +150,16 @@ export class StepFunctionContextService { return null; } - const redrivePostfix = this.context.redrive_count === "0" ? "" : `#${this.context.redrive_count}`; + const countsSuffix = + this.context.retry_count !== "0" || this.context.redrive_count !== "0" + ? `#${this.context.retry_count}#${this.context.redrive_count}` + : ""; const parentId = this.deterministicSha256HashToBigIntString( - this.context.execution_id + - "#" + - this.context.state_name + - "#" + - this.context.state_entered_time + - redrivePostfix, + `${this.context.execution_id}#${this.context.state_name}#${this.context.state_entered_time}${countsSuffix}`, PARENT_ID, ); + const sampleMode = SampleMode.AUTO_KEEP; try { @@ -209,6 +219,7 @@ export class StepFunctionContextService { private extractStateMachineContext(event: any): { execution_id: string; redrive_count: string; + retry_count: string; state_entered_time: string; state_name: string; } | null { @@ -216,6 +227,7 @@ export class StepFunctionContextService { return { execution_id: event.Execution.Id, redrive_count: (event.Execution.RedriveCount ?? "0").toString(), + retry_count: (event.State.RetryCount ?? "0").toString(), state_entered_time: event.State.EnteredTime, state_name: event.State.Name, }; diff --git a/yarn.lock b/yarn.lock index 2629a331..5a83a388 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2816,10 +2816,10 @@ dc-polyfill@^0.1.3, dc-polyfill@^0.1.4: resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.6.tgz#c2940fa68ffb24a7bf127cc6cfdd15b39f0e7f02" integrity sha512-UV33cugmCC49a5uWAApM+6Ev9ZdvIUMTrtCO9fj96TPGOQiea54oeO3tiEVdVeo3J9N2UdJEmbS4zOkkEA35uQ== -dd-trace@^5.37.1: - version "5.37.1" - resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.37.1.tgz#d92af8bc7819b3d12efc43b78ef8764632eff631" - integrity sha512-RiaP1N2jgt+XEw1+Wx7I3FVpP+lOIkLxq08Li27nZYtWE1oa2fN3TLFOsxW47BzaCTg5gWJfGdLsEqUR9Od/Ew== +dd-trace@^5.40.0: + version "5.40.0" + resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.40.0.tgz#42633550bb015a8cf752587f8829fa0c7ae0be15" + integrity sha512-/UYVCcgpZ9LnnUvIJcNfd1Hj51i8HhqLOn9PCj5gK3wJUn6MY/ie/5da2ZaFtoK2DKQ9OZmFBITLV3+KDl4pjA== dependencies: "@datadog/libdatadog" "^0.4.0" "@datadog/native-appsec" "8.4.0"