8000 Make layer publishing idempotent (#118) · DataDog/datadog-lambda-python@85d4fa7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 85d4fa7

Browse files
tianchutianchu
authored
Make layer publishing idempotent (#118)
1 parent 5a0a9dd commit 85d4fa7

File tree

5 files changed

+101
-103
lines changed

5 files changed

+101
-103
lines changed

‎CONTRIBUTING.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ We love pull requests. Here's a quick guide.
2222
./scripts/build_layers.sh
2323
2424
# Publish the a testing layer to your own AWS account, and the ARN will be returned
25-
# Example: ./scripts/publish_layers.sh us-east-1 Datadog-Python37
26-
./scripts/publish_layers.sh <AWS_REGION> <Layer_Name>
25+
# Example: VERSION=1 REGIONS=us-east-1 LAYERS=Datadog-Python37 ./scripts/publish_layers.sh
26+
VERSION=<VERSION> REGIONS=<REGION> LAYERS=<LAYER> ./scripts/publish_layers.sh
2727
```
2828
2929
1. Ensure the unit tests pass (install Docker if you haven't):

‎Dockerfile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,8 @@ RUN find ./python/lib/$runtime/site-packages -name \*.pyc -delete
1616

1717
# Remove botocore (40MB) to reduce package size. aws-xray-sdk
1818
# installs it, while it's already provided by the Lambda Runtime.
19-
RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
19+
RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
20+
21+
# Remove profiling (7MB) to reduce package size.
22+
# Continous profiling is not yet supported anyway.
23+
RUN rm -rf ./python/lib/$runtime/site-packages/ddtrace/profiling

‎scripts/publish_layers.sh

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@
66
# Copyright 2019 Datadog, Inc.
77

88
# Publish the datadog python lambda layer across regions, using the AWS CLI
9-
# Usage: publish_layer.sh [region] [layer]
10-
# Specifying the region and layer arg will publish the specified layer to the specified region
9+
# Usage: VERSION=5 REGIONS=us-east-1 LAYERS=Datadog-Python27 publish_layers.sh
10+
# VERSION is required.
1111
set -e
1212

1313
# Makes sure any subprocesses will be terminated with this process
1414
trap "pkill -P $$; exit 1;" INT
1515

1616
PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
1717
LAYER_PATHS=(".layers/datadog_lambda_py2.7.zip" ".layers/datadog_lambda_py3.6.zip" ".layers/datadog_lambda_py3.7.zip" ".layers/datadog_lambda_py3.8.zip")
18-
AVAILABLE_LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
18+
AVAILABLE_LAYERS=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
1919
AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
2020

2121
# Check that the layer files exist
@@ -27,42 +27,58 @@ do
2727
fi
2828
done
2929

30-
# Check region arg
31-
if [ -z "$1" ]; then
32-
echo "Region parameter not specified, running for all available regions."
30+
# Determine the target regions
31+
if [ -z "$REGIONS" ]; then
32+
echo "Region not specified, running for all available regions."
3333
REGIONS=$AVAILABLE_REGIONS
3434
else
35-
echo "Region parameter specified: $1"
36-
if [[ ! "$AVAILABLE_REGIONS" == *"$1"* ]]; then
37-
echo "Could not find $1 in available regions: $AVAILABLE_REGIONS"
35+
echo "Region specified: $REGIONS"
36+
if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then
37+
echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS"
3838
echo ""
3939
echo "EXITING SCRIPT."
4040
exit 1
4141
fi
42-
REGIONS=($1)
4342
fi
4443

45-
echo "Starting publishing layers for regions: $REGIONS"
46-
47-
# Check layer_name arg
48-
if [ -z "$2" ]; then
49-
echo "Layer name parameter not specified, running for all layer names."
50-
LAYER_NAMES=("${AVAILABLE_LAYER_NAMES[@]}")
44+
# Determine the target layers
45+
if [ -z "$LAYERS" ]; then
46+
echo "Layer not specified, running for all layers."
47+
LAYERS=("${AVAILABLE_LAYERS[@]}")
5148
else
52-
echo "Layer name parameter specified: $2"
53-
if [[ ! " ${AVAILABLE_LAYER_NAMES[@]} " =~ " ${2} " ]]; then
54-
echo "Could not find $2 in available layer names: ${AVAILABLE_LAYER_NAMES[@]}"
49+
echo "Layer specified: $LAYERS"
50+
if [[ ! " ${AVAILABLE_LAYERS[@]} " =~ " ${LAYERS} " ]]; then
51+
echo "Could not find $LAYERS in available layers: ${AVAILABLE_LAYERS[@]}"
5552
echo ""
5653
echo "EXITING SCRIPT."
5754
exit 1
5855
fi
59-
LAYER_NAMES=($2)
6056
fi
6157

58+
# Determine the target layer version
59+
if [ -z "$VERSION" ]; then
60+
echo "Layer version not specified"
61+
echo ""
62+
echo "EXITING SCRIPT."
63+
exit 1
64+
else
65+
echo "Layer version specified: $VERSION"
66+
fi
6267

68+
read -p "Ready to publish version $VERSION of layers ${LAYERS[*]} to regions ${REGIONS[*]} (y/n)?" CONT
69+
if [ "$CONT" != "y" ]; then
70+
echo "Exiting"
71+
exit 1
72+
fi
6373

64-
65-
echo "Publishing layers: ${LAYER_NAMES[*]}"
74+
index_of_layer() {
75+
layer_name=$1
76+
for i in "${!AVAILABLE_LAYERS[@]}"; do
77+
if [[ "${AVAILABLE_LAYERS[$i]}" = "${layer_name}" ]]; then
78+
echo "${i}";
79+
fi
80+
done
81+
}
6682

6783
publish_layer() {
6884
region=$1
@@ -76,46 +92,50 @@ publish_layer() {
7692
--compatible-runtimes $aws_version_key \
7793
| jq -r '.Version')
7894

79-
aws lambda add-layer-version-permission --layer-name $layer_name \
95+
permission=$(aws lambda add-layer-version-permission --layer-name $layer_name \
8096
--version-number $version_nbr \
8197
--statement-id "release-$version_nbr" \
8298
--action lambda:GetLayerVersion --principal "*" \
83-
--region $region
99+
--region $region)
84100

85-
echo "Published layer for region $region, python version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
86-
}
87-
88-
BATCH_SIZE=1
89-
PIDS=()
90-
91-
wait_for_processes() {
92-
for pid in "${PIDS[@]}"; do
93-
wait $pid
94-
done
95-
PIDS=()
101+
echo $version_nbr
96102
}
97103

98104
for region in $REGIONS
99105
do
100106
echo "Starting publishing layer for region $region..."
101-
102107
# Publish the layers for each version of python
103-
i=0
104-
for layer_name in "${LAYER_NAMES[@]}"; do
105-
aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$i]}"
106-
layer_path="${LAYER_PATHS[$i]}"
107-
108-
publish_layer $region $layer_name $aws_version_key $layer_path &
109-
PIDS+=($!)
110-
if [ ${#PIDS[@]} -eq $BATCH_SIZE ]; then
111-
wait_for_processes
108+
for layer_name in "${LAYERS[@]}"; do
109+
latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`')
110+
if [ $latest_version -ge $VERSION ]; then
111+
echo "Layer $layer_name version $VERSION already exists in region $region, skipping..."
112+
continue
113+
elif [ $latest_version -lt $((VERSION-1)) ]; then
114+
read -p "WARNING: The latest version of layer $layer_name in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT
115+
if [ "$CONT" != "y" ]; then
116+
echo "Exiting"
117+
exit 1
118+
fi
112119
fi
113120

114-
i=$(expr $i + 1)
115-
121+
index=$(index_of_layer $layer_name)
122+
aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}"
123+
layer_path="${LAYER_PATHS[$index]}"
124+
125+
while [ $latest_version -lt $VERSION ]; do
126+
latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path)
127+
echo "Published version $latest_version for layer $layer_name in region $region"
128+
129+
# This shouldn't happen unless someone manually deleted the latest version, say 28
130+
# and then try to republish it again. The published version is actually be 29, because
131+
# Lambda layers are immutable and AWS will skip deleted version and use the next number.
132+
if [ $latest_version -gt $VERSION ]; then
133+
echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!"
134+
echo "Exiting"
135+
exit 1
136+
fi
137+
done
116138
done
117139
done
118140

119-
wait_for_processes
120-
121141
echo "Done !"

‎scripts/publish_prod.sh

Lines changed: 23 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
# Use with `aws-vault exec <PROFILE> -- ./publish_prod.sh <DESIRED_NEW_VERSION>
3+
# Use with `./publish_prod.sh <DESIRED_NEW_VERSION>
44

55
set -e
66

@@ -14,8 +14,8 @@ else
1414
git pull origin main
1515
fi
1616

17-
# # Ensure no uncommitted changes
18-
if [ -n "$(git status --porcelain)" ]; then
17+
# Ensure no uncommitted changes
18+
if [ -n "$(git status --porcelain)" ]; then
1919
echo "Detected uncommitted changes, aborting"
2020
exit 1
2121
fi
@@ -37,44 +37,27 @@ AWS_PROFILE=govcloud-us1-fed-human-engineering aws sts get-caller-identity
3737
aws-vault exec prod-engineering -- aws sts get-caller-identity
3838

3939
# Ensure pypi registry access
40-
read -p "Do you have the PyPi login credentials for datadog account (y/n)? " -n 1 -r
41-
echo
42-
if [[ ! $REPLY =~ ^[Yy]$ ]]
43-
then
44-
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
45-
fi
46-
47-
echo 'Checking existing layers in commercial AWS regions'
48-
aws-vault exec prod-engineering -- ./scripts/list_layers.sh
49-
50-
echo 'Checking existing layers in GovCloud AWS regions'
51-
saml2aws login -a govcloud-us1-fed-human-engineering
52-
AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh
53-
54-
read -p "Do the layer lists look good? Proceed publishing the new version (y/n)? " -n 1 -r
55-
echo
56-
if [[ ! $REPLY =~ ^[Yy]$ ]]
57-
then
58-
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
40+
read -p "Do you have the PyPi login credentials for datadog account (y/n)?" CONT
41+
if [ "$CONT" != "y" ]; then
42+
echo "Exiting"
43+
exit 1
5944
fi
6045

6146
VERSION_LINE=$(sed -E -n 's/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/"\1.\2.\3"/p' ./datadog_lambda/__init__.py)
6247
CURRENT_VERSION=$(echo "$VERSION_LINE" | cut -d '"' -f 2)
48+
LAYER_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2)
6349

64-
read -p "Ready to publish layers and update the version from $CURRENT_VERSION to $NEW_VERSION? (y/n)" -n 1 -r
65-
echo
66-
if [[ ! $REPLY =~ ^[Yy]$ ]]
67-
then
68-
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
50+
read -p "Ready to update the library version from $CURRENT_VERSION to $NEW_VERSION and publish layer version $LAYER_VERSION (y/n)?" CONT
51+
if [ "$CONT" != "y" ]; then
52+
echo "Exiting"
53+
exit 1
6954
fi
7055

7156
echo
7257
echo "Replacing __version__ in ./datadog_lambda/__init__.py"
7358
echo
7459
sed -i "" -E "s/\"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\"/\"$NEW_VERSION\"/" ./datadog_lambda/__init__.py
7560

76-
git commit ./datadog_lambda/__init__.py -m "Update module version to ${NEW_VERSION}"
77-
7861
echo
7962
echo "Building layers..."
8063
./scripts/build_layers.sh
@@ -85,25 +68,16 @@ aws-vault exec prod-engineering -- ./scripts/sign_layers.sh prod
8568

8669
echo
8770
echo "Publishing layers to commercial AWS regions"
88-
aws-vault exec prod-engineering -- ./scripts/publish_layers.sh
71+
VERSION=$LAYER_VERSION aws-vault exec prod-engineering -- ./scripts/publish_layers.sh
8972

9073
echo "Publishing layers to GovCloud AWS regions"
9174
saml2aws login -a govcloud-us1-fed-human-engineering
92-
AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh
93-
94-
echo 'Checking published layers in commercial AWS regions'
95-
aws-vault exec prod-engineering -- ./scripts/list_layers.sh
96-
97-
echo 'Checking published layers in GovCloud AWS regions'
98-
saml2aws login -a govcloud-us1-fed-human-engineering
99-
AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/list_layers.sh
75+
VERSION=$LAYER_VERSION AWS_PROFILE=govcloud-us1-fed-human-engineering ./scripts/publish_layers.sh
10076

101-
102-
read -p "Do the layer lists look good? Ready to publish $NEW_VERSION to Pypi? (y/n)" -n 1 -r
103-
echo
104-
if [[ ! $REPLY =~ ^[Yy]$ ]]
105-
then
106-
[[ "$0" = "$BASH_SOURCE" ]] && exit 1 || return 1
77+
read -p "Ready to publish $NEW_VERSION to PyPI (y/n)?" CONT
78+
if [ "$CONT" != "y" ]; then
79+
echo "Exiting"
80+
exit 1
10781
fi
10882

10983
echo
@@ -112,11 +86,11 @@ echo "Publishing to https://pypi.org/project/datadog-lambda/"
11286

11387
echo
11488
echo 'Publishing updates to github'
115-
MINOR_VERSION=$(echo $NEW_VERSION | cut -d '.' -f 2)
89+
git commit ./datadog_lambda/__init__.py -m "Update module version to ${NEW_VERSION}"
11690
git push origin main
117-
git tag "v$MINOR_VERSION"
118-
git push origin "refs/tags/v$MINOR_VERSION"
91+
git tag "v$LAYER_VERSION"
92+
git push origin "refs/tags/v$LAYER_VERSION"
11993

12094
echo
121-
echo "Now create a new release with the tag v${MINOR_VERSION} created"
122-
echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$MINOR_VERSION&title=v$MINOR_VERSION"
95+
echo "Now create a new release with the tag v${LAYER_VERSION} created"
96+
echo "https://github.com/DataDog/datadog-lambda-python/releases/new?tag=v$LAYER_VERSION&title=v$LAYER_VERSION"

‎scripts/publish_sandbox.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
set -e
33

44
./scripts/build_layers.sh
5-
./scripts/sign_layers.sh sandbox
6-
./scripts/publish_layers.sh sa-east-1
5+
aws-vault exec sandbox-account-admin -- ./scripts/sign_layers.sh sandbox
6+
aws-vault exec sandbox-account-admin -- ./scripts/publish_layers.sh

0 commit comments

Comments
 (0)
0