10000 Add script to check AWS regions missing lambda layers (#73) · DataDog/datadog-lambda-python@f6a3022 · GitHub
[go: up one dir, main page]
Skip to content

Commit f6a3022

Browse files
hghotrahghotra
authored
Add script to check AWS regions missing lambda layers (#73)
* Script to copy layers to new region * Update list layers to output regions missing layers * Run list layers before publishing to prod * Fix condition logic * Fix typo * Return 0 if no version found for region * Make script executable
1 parent 0aa6b3e commit f6a3022

File tree

3 files changed

+107
-8
lines changed

3 files changed

+107
-8
lines changed

‎scripts/add_new_region.sh

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2019 Datadog, Inc.
7+
8+
# Copy layers from us-east-1 to new region
9+
# args: [new-region]
10+
11+
set -e
12+
13+
OLD_REGION='us-east-1'
14+
15+
PYTHON_VERSIONS_FOR_AWS_CLI=("python2.7" "python3.6" "python3.7" "python3.8")
16+
LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
17+
NEW_REGION=$1
18+
19+
publish_layer() {
20+
region=$1
21+
layer_name=$2
22+
aws_version_key=$3
23+
layer_path=$4
24+
25+
version_nbr=$(aws lambda publish-layer-version --layer-name $layer_name \
26+
--description "Datadog Lambda Layer for Python" \
27+
--zip-file "fileb://$layer_path" \
28+
--region $region \
29+
--compatible-runtimes $aws_version_key \
30+
| jq -r '.Version')
31+
32+
aws lambda add-layer-version-permission --layer-name $layer_name \
33+
--version-number $version_nbr \
34+
--statement-id "release-$version_nbr" \
35+
--action lambda:GetLayerVersion --principal "*" \
36+
--region $region
37+
38+
echo "Published layer for region $region, version $aws_version_key, layer_name $layer_name, layer_version $version_nbr"
39+
}
40+
41+
get_max_version() {
42+
layer_name=$1
43+
region=$2
44+
last_layer_version=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] | .Version")
45+
if [ "$last_layer_version" == "null" ]; then
46+
echo 0
47+
else
48+
echo $last_layer_version
49+
fi
50+
}
51+
52+
if [ -z "$1" ]; then
53+
echo "Region parameter not specified, exiting"
54+
exit 1
55+
fi
56+
57+
j=0
58+
for layer_name in "${LAYER_NAMES[@]}"; do
59+
# get latest version
60+
last_layer_version=$(get_max_version $layer_name $OLD_REGION)
61+
starting_version=$(get_max_version $layer_name $NEW_REGION)
62+
starting_version=$(expr $starting_version + 1)
63+
64+
# exit if region is already all caught up
65+
if [ $starting_version -gt $last_layer_version ]; then
66+
echo "INFO: $NEW_REGION is already up to date for $layer_name"
67+
continue
68+
fi
69+
70+
# run for each version of layer
71+
for i in $(seq 1 $last_layer_version); do
72+
layer_path=$layer_name"_"$i.zip
73+
aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$j]}"
74+
75+
# download layer versions
76+
URL=$(AWS_REGION=$OLD_REGION aws lambda get-layer-version --layer-name $layer_name --version-number $i --query Content.Location --output text)
77+
curl $URL -o $layer_path
78+
79+
# publish layer to new region
80+
publish_layer $NEW_REGION $layer_name $aws_version_key $layer_path
81+
rm $layer_path
82+
done
83+
84+
j=$(expr $j + 1)
85+
done

‎scripts/list_layers.sh

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,24 @@
88
# Lists most recent layers ARNs across regions to STDOUT
99
# Optionals args: [layer-name] [region]
1010

11+
set -e
12+
1113
LAYER_NAMES=("Datadog-Python27" "Datadog-Python36" "Datadog-Python37" "Datadog-Python38")
12-
AVAILABLE_REGIONS=(us-east-2 us-east-1 us-west-1 us-west-2 ap-east-1 ap-south-1 ap-northeast-2 ap-southeast-1 ap-southeast-2 ap-northeast-1 ca-central-1 eu-north-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 sa-east-1)
14+
AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
15+
LAYERS_MISSING_REGIONS=()
1316

1417
# Check region arg
1518
if [ -z "$2" ]; then
1619
>&2 echo "Region parameter not specified, running for all available regions."
17-
REGIONS=("${AVAILABLE_REGIONS[@]}")
20+
REGIONS=$AVAILABLE_REGIONS
1821
else
22+
1923
>&2 echo "Region parameter specified: $2"
20-
if [[ ! " ${AVAILABLE_REGIONS[@]} " =~ " ${2} " ]]; then
21-
>&2 echo "Could not find $2 in available regions: ${AVAILABLE_REGIONS[@]}"
24+
if [[ ! "$AVAILABLE_REGIONS" == *"$2"* ]]; then
25+
>&2 echo "Could not find $2 in available regions:" $AVAILABLE_REGIONS
2226
>&2 echo ""
2327
>&2 echo "EXITING SCRIPT."
24-
return 1
28+
exit 1
2529
fi
2630
REGIONS=($2)
2731
fi
@@ -41,15 +45,24 @@ else
4145
LAYERS=($1)
4246
fi
4347

44-
for region in "${REGIONS[@]}"
48+
for region in $REGIONS
4549
do
4650
for layer_name in "${LAYERS[@]}"
4751
do
4852
last_layer_arn=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] | .LayerVersionArn")
49-
if [ -z $last_layer_arn ]; then
50-
>&2 echo "No layer found for $region, $layer_name"
53+
if [ "$last_layer_arn" == "null" ]; then
54+
>&2 echo "No layer found for $region, $layer_name"
55+
if [[ ! " ${LAYERS_MISSING_REGIONS[@]} " =~ " ${region} " ]]; then
56+
LAYERS_MISSING_REGIONS+=( $region )
57+
fi
5158
else
5259
echo $last_layer_arn
5360
fi
5461
done
5562
done
63+
64+
if [ ${#LAYERS_MISSING_REGIONS[@]} -gt 0 ]; then
65+
echo "WARNING: Following regions missing layers: ${LAYERS_MISSING_REGIONS[@]}"
66+
echo "Please run ./add_new_region.sh <new_region> to add layers to the missing regions"
67+
exit 1
68+
fi

‎scripts/publish_staging.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/bin/bash
22
set -e
33

4+
./scripts/list_layers.sh
45
./scripts/build_layers.sh
56
./scripts/publish_layers.sh us-east-1

0 commit comments

Comments
 (0)
0