8000 Merge branch 'supabase:develop' into develop · greboid/postgres@b2aad8a · GitHub
[go: up one dir, main page]
Skip to content

Commit b2aad8a

Browse files
greboidgreboid
authored
Merge branch 'supabase:develop' into develop
2 parents 55e421a + e9584d5 commit b2aad8a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+430
-246
lines changed

‎.github/workflows/ami-release.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,31 @@ jobs:
1717
- name: Build AMI
1818
run: |
1919
packer build -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" amazon-arm64.pkr.hcl
20+
21+
- name: Merging migration files
22+
run: cat $(ls -1) > ../migration-output.sql
23+
working-directory: ${{ github.workspace }}/migrations/db/migrations
24+
25+
- name: Push migration files to S3
26+
uses: jakejarvis/s3-sync-action@master
27+
with:
28+
args: --delete
29+
env:
30+
AWS_S3_BUCKET: ${{ secrets.PG_INIT_SCRIPT_S3_BUCKET_STAGING }}
31+
AWS_ACCESS_KEY_ID: ${{ secrets.PG_INIT_SCRIPT_ACCESS_KEY_ID_STAGING }}
32+
AWS_SECRET_ACCESS_KEY: ${{ secrets.PG_INIT_SCRIPT_SECRET_ACCESS_KEY_STAGING }}
33+
AWS_REGION: ap-southeast-1
34+
SOURCE_DIR: migrations/db
35+
DEST_DIR: migrations/db
36+
37+
- name: Push migration files to S3
38+
uses: jakejarvis/s3-sync-action@master
39+
with:
40+
args: --delete
41+
env:
42+
AWS_S3_BUCKET: ${{ secrets.PG_INIT_SCRIPT_S3_BUCKET_PROD }}
43+
AWS_ACCESS_KEY_ID: ${{ secrets.PG_INIT_SCRIPT_ACCESS_KEY_ID_PROD }}
44+
AWS_SECRET_ACCESS_KEY: ${{ secrets.PG_INIT_SCRIPT_SECRET_ACCESS_KEY_PROD }}
45+
AWS_REGION: ap-southeast-1
46+
SOURCE_DIR: migrations/db
47+
DEST_DIR: migrations/db

‎.github/workflows/dockerhub-release.yml

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ jobs:
3434
with:
3535
images: |
3636
supabase/postgres
37-
public.ecr.aws/t3w2s2c9/postgres
3837
tags: |
3938
type=raw,value=${{ needs.settings.outputs.docker_version }}_${{ env.arch }}
4039
@@ -45,13 +44,6 @@ jobs:
4544
username: ${{ secrets.DOCKER_USERNAME }}
4645
password: ${{ secrets.DOCKER_PASSWORD }}
4746

48-
- name: Login to ECR
49-
uses: docker/login-action@v2
50-
with:
51-
registry: public.ecr.aws
52-
username: ${{ secrets.PROD_ACCESS_KEY_ID }}
53-
password: ${{ secrets.PROD_SECRET_ACCESS_KEY }}
54-
5547
- id: build
5648
uses: docker/build-push-action@v3
5749
with:
@@ -77,7 +69,6 @@ jobs:
7769
with:
7870
images: |
7971
supabase/postgres
80-
public.ecr.aws/t3w2s2c9/postgres
8172
tags: |
8273
type=raw,value=${{ needs.settings.outputs.docker_version }}_${{ env.arch }}
8374
@@ -86,13 +77,6 @@ jobs:
8677
username: ${{ secrets.DOCKER_USERNAME }}
8778
password: ${{ secrets.DOCKER_PASSWORD }}
8879

89-
- name: Login to ECR
90-
uses: docker/login-action@v2
91-
with:
92-
registry: public.ecr.aws
93-
username: ${{ secrets.PROD_ACCESS_KEY_ID }}
94-
password: ${{ secrets.PROD_SECRET_ACCESS_KEY }}
95-
9680
- uses: docker/setup-buildx-action@v2
9781
with:
9882
driver: docker
@@ -112,6 +96,9 @@ jobs:
11296
merge_manifest:
11397
needs: [settings, docker_x86_release, docker_arm_release]
11498
runs-on: ubuntu-latest
99+
permissions:
100+
contents: read
101+
packages: write
115102
steps:
116103
- uses: docker/setup-buildx-action@v2
117104

@@ -133,8 +120,17 @@ jobs:
133120
username: ${{ secrets.PROD_ACCESS_KEY_ID }}
134121
password: ${{ secrets.PROD_SECRET_ACCESS_KEY }}
135122

136-
- name: Mirror to ECR
137-
uses: akhilerm/tag-push-action@v2.0.0
123+
- name: Login to GHCR
124+
uses: docker/login-action@v2
125+
with:
126+
registry: ghcr.io
127+
username: ${{ github.actor }}
128+
password: ${{ secrets.GITHUB_TOKEN }}
129+
130+
- name: Mirror Images
131+
uses: akhilerm/tag-push-action@v2.1.0
138132
with:
139133
src: docker.io/supabase/postgres:${{ needs.settings.outputs.docker_version }}
140-
dst: public.ecr.aws/t3w2s2c9/postgres:${{ needs.settings.outputs.docker_version }}
134+
dst: |
135+
public.ecr.aws/supabase/postgres:${{ needs.settings.outputs.docker_version }}
136+
ghcr.io/supabase/postgres:${{ needs.settings.outputs.docker_version }}

‎.github/workflows/mirror.yml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Mirror to ECR
1+
name: Mirror Image
22

33
on:
44
workflow_dispatch:
@@ -11,13 +11,23 @@ on:
1111
jobs:
1212
mirror:
1313
runs-on: ubuntu-latest
14+
permissions:
15+
contents: read
16+
packages: write
1417
steps:
1518
- uses: docker/login-action@v2
1619
with:
1720
registry: public.ecr.aws
1821
username: ${{ secrets.PROD_ACCESS_KEY_ID }}
1922
password: ${{ secrets.PROD_SECRET_ACCESS_KEY }}
20-
- uses: akhilerm/tag-push-action@v2.0.0
23+
- uses: docker/login-action@v2
24+
with:
25+
registry: ghcr.io
26+
username: ${{ github.actor }}
27+
password: ${{ secrets.GITHUB_TOKEN }}
28+
- uses: akhilerm/tag-push-action@v2.1.0
2129
with:
2230
src: docker.io/supabase/postgres:${{ inputs.version }}
23-
dst: public.ecr.aws/t3w2s2c9/postgres:${{ inputs.version }}
31+
dst: |
32+
public.ecr.aws/supabase/postgres:${{ inputs.version }}
33+
ghcr.io/supabase/postgres:${{ inputs.version }}

‎amazon-arm.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"variables": {
33
"profile": "{{env `AWS_PROFILE`}}",
44
"region": "ap-northeast-1",
5-
"ami_regions": "eu-central-1,eu-west-1,eu-west-2,ap-south-1,ap-southeast-1,ap-southeast-2,us-west-1,us-east-1,ca-central-1,sa-east-1,ap-northeast-1,ap-northeast-2",
5+
"ami_regions": "eu-central-1,eu-west-1,eu-west-2,eu-west-3,ap-south-1,ap-southeast-1,ap-southeast-2,us-west-1,us-east-1,ca-central-1,sa-east-1,ap-northeast-1,ap-northeast-2",
66
"ami": "",
77
"ami_name": "",
88
"environment": "prod",

‎ansible/files/admin_api_scripts/pg_upgrade_check.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#! /usr/bin/env bash
2+
## This script provides a method to check the status of the database upgrade
3+
## process, which is updated in /tmp/pg-upgrade-status by pg_upgrade_initiate.sh
4+
## This runs on the old (source) instance.
5+
6+
set -euo pipefail
7+
8+
STATUS_FILE="/tmp/pg-upgrade-status"
9+
10+
if [ -f "${STATUS_FILE}" ]; then
11+
STATUS=$(cat "${STATUS_FILE}")
12+
echo -n "${STATUS}"
13+
else
14+
echo -n "unknown"
15+
fi
16+

‎ansible/files/admin_api_scripts/pg_upgrade_complete.sh

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#! /usr/bin/env bash
2+
3+
## This script is run on the newly launched instance which is to be promoted to
4+
## become the primary database instance once the upgrade successfully completes.
5+
## The following commands copy custom PG configs and enable previously disabled
6+
## extensions, containing regtypes referencing system OIDs.
7+
8+
# Extensions to be reenabled after pg_upgrade.
9+
# Running an upgrade with these extensions enabled will result in errors due to
10+
# them depending on regtypes referencing system OIDs. Thus they have been disabled
11+
# beforehand.
12+
EXTENSIONS_TO_REENABLE=(
13+
"pg_graphql"
14+
)
15+
16+
17+
run_sql() {
18+
STATEMENT=$1
19+
psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT"
20+
}
21+
22+
function complete_pg_upgrade {
23+
mount -a -v
24+
25+
# copying custom configurations
26+
cp /data/conf/* /etc/postgresql-custom/
27+
28+
service postgresql start
29+
su -c 'vacuumdb --all --analyze-in-stages' -s $SHELL postgres
30+
31+
for EXTENSION in "${EXTENSIONS_TO_REENABLE[@]}"; do
32+
run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE;"
33+
done
34+
35+
sleep 5
36+
service postgresql restart
37+
38+
sleep 5
39+
service postgresql restart
40+
}
41+
42+
set -euo pipefail
43+
44+
complete_pg_upgrade >> /var/log/pg-upgrade-complete.log 2>&1
45+
echo "Upgrade job completed"

‎ansible/files/admin_api_scripts/pg_upgrade_initiate.sh

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#! /usr/bin/env bash
2+
3+
## This script is run on the old (source) instance, mounting the data disk
4+
## of the newly launched instance, disabling extensions containing regtypes,
5+
## and running pg_upgrade.
6+
## It reports the current status of the upgrade process to /tmp/pg-upgrade-status,
7+
## which can then be subsequently checked through pg_upgrade_check.sh.
8+
9+
# Extensions to disable before running pg_upgrade.
10+
# Running an upgrade with these extensions enabled will result in errors due to
11+
# them depending on regtypes referencing system OIDs.
12+
EXTENSIONS_TO_DISABLE=(
13+
"pg_graphql"
14+
)
15+
16+
set -eEuo pipefail
17+
18+
PGVERSION=$1
19+
20+
MOUNT_POINT="/data_migration"
21+
22+
run_sql() {
23+
STATEMENT=$1
24+
psql -h localhost -U supabase_admin -d postgres -c "$STATEMENT"
25+
}
26+
27+
cleanup() {
28+
UPGRADE_STATUS=${1:-"failed"}
29+
EXIT_CODE=${?:-0}
30+
31+
systemctl start postgresql
32+
33+
for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do
34+
run_sql "CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCDE;"
35+
done
36+
37+
run_sql "ALTER USER postgres WITH NOSUPERUSER;"
38+
39+
umount $MOUNT_POINT
40+
echo "${UPGRADE_STATUS}" > /tmp/pg-upgrade-status
41+
42+
exit $EXIT_CODE
43+
}
44+
45+
function initiate_upgrade {
46+
BLOCK_DEVICE=$(lsblk -dpno name | grep -v "/dev/nvme[0-1]")
47+
echo "running" > /tmp/pg-upgrade-status
48+
49+
mkdir -p "$MOUNT_POINT"
50+
mount "$BLOCK_DEVICE" "$MOUNT_POINT"
51+
52+
mkdir -p "/tmp/pg_upgrade_bin"
53+
tar zxvf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin"
54+
chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION"
55+
56+
for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do
57+
run_sql "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;"
58+
done
59+
60+
run_sql "ALTER USER postgres WITH SUPERUSER;"
61+
62+
PGDATAOLD=$(cat /etc/postgresql/postgresql.conf | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/");
63+
PGDATANEW="$MOUNT_POINT/pgdata"
64+
PGBINNEW="/tmp/pg_upgrade_bin/$PGVERSION/bin"
65+
PGSHARENEW="/tmp/pg_upgrade_bin/$PGVERSION/share"
66+
67+
chown -R postgres:postgres "$MOUNT_POINT/"
68+
rm -rf "$PGDATANEW/"
69+
su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s $SHELL postgres
70+
71+
# running upgrade using at least 1 cpu core
72+
WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }')
73+
74+
# upgrade job outputs a log in the cwd; needs write permissions
75+
cd /tmp
76+
77+
UPGRADE_COMMAND=$(cat <<EOF
78+
time ${PGBINNEW}/pg_upgrade \
79+
--old-bindir="/usr/lib/postgresql/bin" \
80+
--new-bindir=${PGBINNEW} \
81+
--old-datadir=${PGDATAOLD} \
82+
--new-datadir=${PGDATANEW} \
83+
--jobs="${WORKERS}" \
84+
--old-options='-c config_file=/etc/postgresql/postgresql.conf' \
85+
--new-options="-c data_directory=${PGDATANEW}"
86+
EOF
87+
)
88+
89+
systemctl stop postgresql
90+
su -c "$UPGRADE_COMMAND" -s $SHELL postgres
91+
92+
# copying custom configurations
93+
mkdir -p $MOUNT_POINT/conf
94+
cp /etc/postgresql-custom/* $MOUNT_POINT/conf/
95+
96+
cleanup "complete"
97+
}
98+
99+
trap cleanup ERR
100+
101+
initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1
102+
echo "Upgrade initiate job completed "

‎ansible/files/admin_api_scripts/pg_upgrade_prepare.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#! /usr/bin/env bash
2+
## This script is runs in advance of the database version upgrade, on the newly
3+
## launched instance which will eventually be promoted to become the primary
4+
## database instance once the upgrade successfully completes, terminating the
5+
## previous (source) instance.
6+
## The following commands safely stop the Postgres service and unmount
7+
## the data disk off the newly launched instance, to be re-attached to the
8+
## source instance and run the upgrade there.
9+
10+
set -euo pipefail
11+
12+
systemctl stop postgresql
13+
umount /data
14+

‎ansible/files/adminapi.sudoers.conf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,16 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st
1010
%adminapi ALL= NOPASSWD: /root/disable_walg.sh
1111
%adminapi ALL= NOPASSWD: /root/enable_walg.sh
1212
%adminapi ALL= NOPASSWD: /root/manage_readonly_mode.sh
13+
%adminapi ALL= NOPASSWD: /root/pg_upgrade_prepare.sh
14+
%adminapi ALL= NOPASSWD: /root/pg_upgrade_initiate.sh
15+
%adminapi ALL= NOPASSWD: /root/pg_upgrade_complete.sh
16+
%adminapi ALL= NOPASSWD: /root/pg_upgrade_check.sh
1317
%adminapi ALL= NOPASSWD: /usr/bin/systemctl daemon-reload
1418
%adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service
1519
%adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service
1620
%adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload
1721
%adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice
22+
%adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf
1823
%adminapi ALL= NOPASSWD: KONG
1924
%adminapi ALL= NOPASSWD: POSTGREST
2025
%adminapi ALL= NOPASSWD: GOTRUE

‎ansible/files/fail2ban_config/fail2ban.service.conf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
1+
[Unit]
2+
After=nftables.service
3+
Wants=nftables.service
4+
15
[Service]
26
ExecStartPost=/bin/bash -c "sleep 5 && chmod g+w /var/run/fail2ban/fail2ban.sock"

0 commit comments

Comments
 (0)
0