diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 2f1c9a1ea..000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,2 +0,0 @@ -* @supabase/backend -migrations/ @supabase/cli @supabase/backend diff --git a/.github/workflows/ami-release.yml b/.github/workflows/ami-release.yml deleted file mode 100644 index 209894de7..000000000 --- a/.github/workflows/ami-release.yml +++ /dev/null @@ -1,154 +0,0 @@ -name: Release AMI - -on: - push: - branches: - - develop - paths: - - '.github/workflows/ami-release.yml' - - 'common.vars.pkr.hcl' - workflow_dispatch: - -jobs: - build: - strategy: - matrix: - include: - - runner: arm-runner - arch: arm64 - ubuntu_release: focal - ubuntu_version: 20.04 - mcpu: neoverse-n1 - runs-on: ${{ matrix.runner }} - timeout-minutes: 150 - permissions: - contents: write - packages: write - id-token: write - - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/build-push-action@v3 - with: - push: false - load: true - build-args: | - ${{ steps.args.outputs.result }} - target: extensions - tags: supabase/postgres:extensions - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Extract built packages - run: | - mkdir -p /tmp/extensions ansible/files/extensions - docker save supabase/postgres:extensions | tar xv -C /tmp/extensions - for layer in /tmp/extensions/*/layer.tar; do - tar xvf "$layer" -C ansible/files/extensions --strip-components 1 - done - - - id: version - run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" - - name: Build Postgres deb - uses: docker/build-push-action@v3 - with: - push: false - load: true - file: docker/Dockerfile - target: pg-deb - build-args: | - ubuntu_release=${{ matrix.ubuntu_release }} - ubuntu_release_no=${{ matrix.ubuntu_version }} - postgresql_major=${{ steps.version.outputs.postgresql_major }} - postgresql_release=${{ steps.version.outputs.postgresql_release }} - CPPFLAGS=-mcpu=${{ matrix.mcpu }} - tags: supabase/postgres:deb - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Extract Postgres deb - run: | - mkdir -p /tmp/build ansible/files/postgres - docker save supabase/postgres:deb | tar xv -C /tmp/build - for layer in /tmp/build/*/layer.tar; do - tar xvf "$layer" -C ansible/files/postgres --strip-components 1 - done - - - name: Build AMI - run: | - GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64.pkr.hcl - - - name: Grab release version - id: process_release_version - run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload software manifest to s3 staging - run: | - cd ansible - ansible-playbook -i localhost \ - -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ - -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ - manifest-playbook.yml - - - name: Upload pg binaries to s3 staging - run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - - - name: configure aws credentials - prod - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload software manifest to s3 prod - run: | - cd ansible - ansible-playbook -i localhost \ - -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ - -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ - manifest-playbook.yml - - - name: Upload pg binaries to s3 prod - run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - - - name: Create release - uses: softprops/action-gh-release@v1 - with: - name: ${{ steps.process_release_version.outputs.version }} - tag_name: ${{ steps.process_release_version.outputs.version }} - target_commitish: ${{github.sha}} - - - name: Slack Notification on Failure - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: 'gha-failures-notifier' - SLACK_COLOR: 'danger' - SLACK_MESSAGE: 'Building Postgres AMI failed' - SLACK_FOOTER: '' - - - name: Cleanup resources on build cancellation - if: ${{ cancelled() }} - run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {} diff --git a/.github/workflows/build-ccache.yml b/.github/workflows/build-ccache.yml deleted file mode 100644 index 888fbbe14..000000000 --- a/.github/workflows/build-ccache.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Update ccache - -on: - push: - branches: - - develop - paths: - - ".github/workflows/build-ccache.yml" - - "ansible/vars.yml" - - "Dockerfile" - workflow_dispatch: - -env: - image_tag: public.ecr.aws/supabase/postgres:ccache -permissions: - contents: read - packages: write - id-token: write - -jobs: - settings: - runs-on: ubuntu-latest - outputs: - build_args: ${{ steps.args.outputs.result }} - steps: - - uses: actions/checkout@v3 - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - build_image: - needs: settings - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} - steps: - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - name: Configure AWS credentials - prod - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" - - uses: docker/login-action@v2 - with: - registry: public.ecr.aws - - id: build - uses: docker/build-push-action@v3 - with: - push: true - target: buildcache - build-args: | - CACHE_EPOCH=${{ github.event.repository.updated_at }} - ${{ needs.settings.outputs.build_args }} - tags: ${{ env.image_tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - - merge_manifest: - needs: build_image - runs-on: ubuntu-latest - steps: - - uses: docker/setup-buildx-action@v2 - - name: Configure AWS credentials - prod - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" - - uses: docker/login-action@v2 - with: - registry: public.ecr.aws - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ env.image_tag }} \ - ${{ env.image_tag }}_amd64 \ - ${{ env.image_tag }}_arm64 diff --git a/.github/workflows/check-shellscripts.yml b/.github/workflows/check-shellscripts.yml deleted file mode 100644 index b796bdb90..000000000 --- a/.github/workflows/check-shellscripts.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Check shell scripts - -on: - push: - branches: - - develop - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 - with: - scandir: './ansible/files/admin_api_scripts' - - - name: Run ShellCheck on pg_upgrade scripts - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: -e SC2001 -e SC2002 -e SC2143 - with: - scandir: './ansible/files/admin_api_scripts/pg_upgrade_scripts' diff --git a/.github/workflows/collect-u18-binaries.yml b/.github/workflows/collect-u18-binaries.yml deleted file mode 100644 index 21116a809..000000000 --- a/.github/workflows/collect-u18-binaries.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: Collect Ubuntu 18 compatible binaries - -on: - push: - branches: - - develop - - pcnc/pg-upgrade-fixes - paths: - - '.github/workflows/collect-u18-binaries.yml' - - 'common.vars.pkr.hcl' - - workflow_run: - workflows: [Release AMI] - branches: - - develop - types: - - completed - - workflow_dispatch: - -jobs: - build: - strategy: - matrix: - include: - - runner: arm-runner - arch: arm64 - ubuntu_release: bionic - ubuntu_version: 18.04 - mcpu: neoverse-n1 - runs-on: ${{ matrix.runner }} - timeout-minutes: 150 - permissions: - contents: write - packages: write - id-token: write - - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/build-push-action@v3 - with: - push: false - load: true - file: Dockerfile-u18 - build-args: | - ${{ steps.args.outputs.result }} - target: extensions - tags: supabase/postgres:extensions-u18 - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max,ignore-error=true - - - name: Extract built packages - run: | - mkdir -p /tmp/extensions ansible/files/extensions - docker save supabase/postgres:extensions-u18 | tar xv -C /tmp/extensions - for layer in /tmp/extensions/*/layer.tar; do - tar xvf "$layer" -C ansible/files/extensions --strip-components 1 - done - - - id: version - run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" - - name: Build Postgres deb - uses: docker/build-push-action@v3 - with: - push: false - load: true - file: docker/Dockerfile - target: pg-deb - build-args: | - ubuntu_release=${{ matrix.ubuntu_release }} - ubuntu_release_no=${{ matrix.ubuntu_version }} - postgresql_major=${{ steps.version.outputs.postgresql_major }} - postgresql_release=${{ steps.version.outputs.postgresql_release }} - DEB_BUILD_PROFILES=pkg.postgresql.nozstd - tags: supabase/postgres:deb-u18 - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max,ignore-error=true - - name: Extract Postgres deb - run: | - mkdir -p /tmp/build ansible/files/postgres - docker save supabase/postgres:deb-u18 | tar xv -C /tmp/build - for layer in /tmp/build/*/layer.tar; do - tar xvf "$layer" -C ansible/files/postgres --strip-components 1 - done - - - name: Build surrogate Docker image - uses: docker/build-push-action@v3 - with: - push: false - load: true - file: Dockerfile-u18 - target: pg_binary_collection - build-args: | - ubuntu_release=${{ matrix.ubuntu_release }} - ubuntu_release_no=${{ matrix.ubuntu_version }} - postgresql_major=${{ steps.version.outputs.postgresql_major }} - postgresql_release=${{ steps.version.outputs.postgresql_release }} - tags: supabase/postgres:u18-binaries - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max,ignore-error=true - - - name: Copy binary tarball - run: | - CONTAINER_ID=$(docker create supabase/postgres:u18-binaries) - docker cp "${CONTAINER_ID}:/tmp/pg_binaries/${{ matrix.ubuntu_version }}.tar.gz" > /tmp/pg_binaries.tar.gz - docker rm "${CONTAINER_ID}" - - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload pg binaries to s3 staging - run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/18.04.tar.gz - - - name: configure aws credentials - prod - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload pg binaries to s3 prod - run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/18.04.tar.gz diff --git a/.github/workflows/dockerhub-release-aio.yml b/.github/workflows/dockerhub-release-aio.yml deleted file mode 100644 index 9da4e13a7..000000000 --- a/.github/workflows/dockerhub-release-aio.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: Release AIO image on Dockerhub - -on: - push: - branches: - - pcnc/trigger-build - paths: - - ".github/workflows/dockerhub-release-aio.yml" - - "docker/all-in-one/*" - - "common.vars*" - workflow_run: - workflows: [Release on Dockerhub] - branches: - - develop - types: - - completed - -jobs: - settings: - runs-on: ubuntu-latest - outputs: - docker_version: ${{ steps.settings.outputs.postgres-version }} - image_tag: supabase/postgres:aio-${{ steps.settings.outputs.postgres-version }} - build_args: ${{ steps.args.outputs.result }} - steps: - - uses: actions/checkout@v3 - - id: settings - # Remove spaces and quotes to get the raw version string - run: sed -r 's/(\s|\")+//g' common.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - build_image: - needs: settings - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} - steps: - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - id: build - uses: docker/build-push-action@v3 - with: - file: docker/all-in-one/Dockerfile - push: true - build-args: | - postgres_version=${{ needs.settings.outputs.docker_version }} - ${{ needs.settings.outputs.build_args }} - target: production - tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres AIO ${{ matrix.arch }} image failed" - SLACK_FOOTER: "" - - merge_manifest: - needs: [settings, build_image] - runs-on: ubuntu-latest - steps: - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ - ${{ needs.settings.outputs.image_tag }}_amd64 \ - ${{ needs.settings.outputs.image_tag }}_arm64 - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed" - SLACK_FOOTER: "" - - publish: - needs: [settings, merge_manifest] - # Call workflow explicitly because events from actions cannot trigger more actions - uses: ./.github/workflows/mirror.yml - with: - version: aio-${{ needs.settings.outputs.docker_version }} - secrets: inherit diff --git a/.github/workflows/dockerhub-release.yml b/.github/workflows/dockerhub-release.yml deleted file mode 100644 index 1f1453c38..000000000 --- a/.github/workflows/dockerhub-release.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Release on Dockerhub - -on: - push: - branches: - - develop - paths: - - ".github/workflows/dockerhub-release.yml" - - "common.vars*" - -jobs: - settings: - runs-on: ubuntu-latest - outputs: - docker_version: ${{ steps.settings.outputs.postgres-version }} - image_tag: supabase/postgres:${{ steps.settings.outputs.postgres-version }} - build_args: ${{ steps.args.outputs.result }} - steps: - - uses: actions/checkout@v3 - - id: settings - # Remove spaces and quotes to get the raw version string - run: sed -r 's/(\s|\")+//g' common.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - build_image: - needs: settings - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} - steps: - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - id: build - uses: docker/build-push-action@v3 - with: - push: true - build-args: | - ${{ needs.settings.outputs.build_args }} - target: production - tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha - cache-to: type=gha,mode=max - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres ${{ matrix.arch }} image failed" - SLACK_FOOTER: "" - - merge_manifest: - needs: [settings, build_image] - runs-on: ubuntu-latest - steps: - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ - ${{ needs.settings.outputs.image_tag }}_amd64 \ - ${{ needs.settings.outputs.image_tag }}_arm64 - - name: Slack Notification - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: "gha-failures-notifier" - SLACK_COLOR: "danger" - SLACK_MESSAGE: "Building Postgres image failed" - SLACK_FOOTER: "" - - publish: - needs: [settings, merge_manifest] - # Call workflow explicitly because events from actions cannot trigger more actions - uses: ./.github/workflows/mirror.yml - with: - version: ${{ needs.settings.outputs.docker_version }} - secrets: inherit diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml deleted file mode 100644 index 9f53e7fd7..000000000 --- a/.github/workflows/mirror.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Mirror Image - -on: - workflow_call: - inputs: - version: - required: true - type: string - workflow_dispatch: - inputs: - version: - description: "Image tag" - required: true - type: string - -jobs: - mirror: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - steps: - - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@v2.2.0 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: us-east-1 - - uses: docker/login-action@v2 - with: - registry: public.ecr.aws - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: akhilerm/tag-push-action@v2.1.0 - with: - src: docker.io/supabase/postgres:${{ inputs.version }} - dst: | - public.ecr.aws/supabase/postgres:${{ inputs.version }} - ghcr.io/supabase/postgres:${{ inputs.version }} diff --git a/.github/workflows/package-plv8.yml b/.github/workflows/package-plv8.yml deleted file mode 100644 index 0944eb9a8..000000000 --- a/.github/workflows/package-plv8.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Package plv8 - -on: - push: - branches: - - develop - paths: - - ".github/workflows/package-plv8.yml" - - "Dockerfile" - -env: - image: ghcr.io/supabase/plv8 -permissions: - contents: read - packages: write - id-token: write - -jobs: - settings: - runs-on: ubuntu-latest - outputs: - image_tag: ${{ env.image }}:${{ steps.meta.outputs.image_tag }} - steps: - - uses: actions/checkout@v3 - - id: meta - run: | - plv8_release=$(grep -o 'plv8_release=.*' Dockerfile | head -1 | cut -d "=" -f 2) - postgresql_major=$(grep -o 'postgresql_major=.*' Dockerfile | head -1 | cut -d "=" -f 2) - echo "image_tag=${plv8_release}-pg${postgresql_major}" >> $GITHUB_OUTPUT - - build_image: - needs: settings - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - outputs: - image_digest: ${{ steps.build.outputs.digest }} - steps: - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - id: build - uses: docker/build-push-action@v3 - with: - push: true - target: plv8-deb - tags: ${{ needs.settings.outputs.image_tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - no-cache: true - - merge_manifest: - needs: [settings, build_image] - runs-on: ubuntu-latest - steps: - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Merge multi-arch manifests - run: | - docker buildx imagetools create -t ${{ needs.settings.outputs.image_tag }} \ - ${{ needs.settings.outputs.image_tag }}_amd64 \ - ${{ needs.settings.outputs.image_tag }}_arm64 diff --git a/.github/workflows/publish-migrations.yml b/.github/workflows/publish-migrations.yml deleted file mode 100644 index 1ad95843a..000000000 --- a/.github/workflows/publish-migrations.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Release Migrations - -on: - push: - branches: - - develop - -jobs: - build: - runs-on: [self-hosted, linux] - timeout-minutes: 15 - - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - - name: Merging migration files - run: cat $(ls -1) > ../migration-output.sql - working-directory: ${{ github.workspace }}/migrations/db/migrations - - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" - - - name: Deploy to S3 staging - shell: bash - run: aws s3 sync migrations/db s3://$AWS_S3_BUCKET/migrations/db --delete - env: - AWS_S3_BUCKET: ${{ secrets.PG_INIT_SCRIPT_S3_BUCKET_STAGING }} - - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "ap-southeast-1" - - - name: Deploy to S3 staging - shell: bash - run: aws s3 sync migrations/db s3://$AWS_S3_BUCKET/migrations/db --delete - env: - AWS_S3_BUCKET: ${{ secrets.PG_INIT_SCRIPT_S3_BUCKET_PROD }} diff --git a/.github/workflows/publish-pgupgrade-scripts.yml b/.github/workflows/publish-pgupgrade-scripts.yml deleted file mode 100644 index 49a74f447..000000000 --- a/.github/workflows/publish-pgupgrade-scripts.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Publish pg_upgrade_scripts - -on: - push: - branches: - - develop - paths: - - '.github/workflows/publish-pgupgrade-scripts.yml' - - 'common.vars.pkr.hcl' - workflow_dispatch: - -permissions: - id-token: write - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - - name: Grab release version - id: process_release_version - run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Create a tarball containing pg_upgrade scripts - run: | - mkdir -p /tmp/pg_upgrade_scripts - cp -r ansible/files/admin_api_scripts/pg_upgrade_scripts/* /tmp/pg_upgrade_scripts - tar -czvf /tmp/pg_upgrade_scripts.tar.gz -C /tmp/ pg_upgrade_scripts - - - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload pg_upgrade scripts to s3 staging - run: | - aws s3 cp /tmp/pg_upgrade_scripts.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz - - - name: configure aws credentials - prod - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" - - - name: Upload pg_upgrade scripts to s3 prod - run: | - aws s3 cp /tmp/pg_upgrade_scripts.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/pg_upgrade_scripts.tar.gz - - - name: Slack Notification on Failure - if: ${{ failure() }} - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - SLACK_USERNAME: 'gha-failures-notifier' - SLACK_COLOR: 'danger' - SLACK_MESSAGE: 'Publishing pg_upgrade scripts failed' - SLACK_FOOTER: '' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 48523ea47..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Test Database - -on: - push: - branches: - - develop - pull_request: - workflow_dispatch: - -jobs: - build: - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 180 - env: - POSTGRES_PORT: 5478 - POSTGRES_PASSWORD: password - steps: - - uses: actions/checkout@v3 - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - - run: docker context create builders - - uses: docker/setup-buildx-action@v2 - with: - endpoint: builders - - uses: docker/build-push-action@v3 - with: - push: false - load: true - context: . - target: production - build-args: | - ${{ steps.args.outputs.result }} - tags: supabase/postgres:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Start Postgres - run: | - docker run --rm --pull=never \ - -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \ - -p ${{ env.POSTGRES_PORT }}:5432 \ - --name supabase_postgres \ - -d supabase/postgres:latest - - - name: Install psql - run: | - sudo apt update - sudo apt install -y --no-install-recommends postgresql-client - - - name: Install pg_prove - run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP - env: - SHELL: /bin/bash - - - name: Wait for healthy database - run: | - count=0 - until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do - exit=$? - count=$((count + 1)) - if [ $count -ge "$retries" ]; then - echo "Retry $count/$retries exited $exit, no more retries left." - docker stop -t 2 "$container" - return $exit - fi - sleep 1; - done; - echo "$container container is healthy" - env: - retries: 20 - container: supabase_postgres - - - name: Run tests - run: pg_prove migrations/tests/test.sql - env: - PGHOST: localhost - PGPORT: ${{ env.POSTGRES_PORT }} - PGDATABASE: postgres - PGUSER: supabase_admin - PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - - - name: Check migrations are idempotent - run: | - for sql in ./migrations/db/migrations/*.sql; do - echo "$0: running $sql" - psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql" - done - env: - PGHOST: localhost - PGPORT: ${{ env.POSTGRES_PORT }} - PGDATABASE: postgres - PGUSER: supabase_admin - PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - - schema: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: verify schema.sql is committed - run: | - docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then - echo "Detected uncommitted changes after build. See status below:" - git diff - exit 1 - fi diff --git a/.github/workflows/testinfra.yml b/.github/workflows/testinfra.yml index bea5c5254..90b3dd910 100644 --- a/.github/workflows/testinfra.yml +++ b/.github/workflows/testinfra.yml @@ -5,25 +5,100 @@ on: workflow_dispatch: jobs: - build: - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 + # test-all-in-one: + # strategy: + # matrix: + # include: + # - runner: [self-hosted, X64] + # arch: amd64 + # - runner: arm-runner + # arch: arm64 + # runs-on: ${{ matrix.runner }} + # timeout-minutes: 30 + # steps: + # - uses: actions/checkout@v3 + + # - run: docker context create builders + # - uses: docker/setup-buildx-action@v3 + # with: + # endpoint: builders + + # - name: Run aio integration tests + # run: | + # pip3 install docker pytest pytest-testinfra + # pytest -vv testinfra/test_all_in_one.py + + test-ami: + runs-on: ubuntu-latest + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write + steps: - - uses: actions/checkout@v3 + - name: Checkout Repo + uses: actions/checkout@v4 + - id: args + uses: mikefarah/yq@master + with: + cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - run: docker context create builders - uses: docker/setup-buildx-action@v3 with: endpoint: builders + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v5 + with: + push: false + load: true + build-args: | + ${{ steps.args.outputs.result }} + target: extensions + tags: soedirgo/postgres:extensions + platforms: linux/amd64 + cache-from: type=registry,ref=ghcr.io/soedirgo/postgres:extensions-cache + cache-to: type=registry,ref=ghcr.io/soedirgo/postgres:extensions-cache,mode=max + # - name: Extract built packages + # run: | + # mkdir -p /tmp/extensions ansible/files/extensions + # docker save supabase/postgres:extensions | tar xv -C /tmp/extensions + # for layer in /tmp/extensions/*/layer.tar; do + # tar xvf "$layer" -C ansible/files/extensions --strip-components 1 + # done + + - id: version + run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" + - name: Build Postgres deb + uses: docker/build-push-action@v5 + with: + push: false + load: true + file: docker/Dockerfile + target: pg-deb + tags: soedirgo/postgres:deb + platforms: linux/amd64 + cache-from: type=registry,ref=ghcr.io/soedirgo/postgres:deb-cache + cache-to: type=registry,ref=ghcr.io/soedirgo/postgres:deb-cache,mode=max + # - name: Extract Postgres deb + # run: | + # mkdir -p /tmp/build ansible/files/postgres + # docker save supabase/postgres:deb | tar xv -C /tmp/build + # for layer in /tmp/build/*/layer.tar; do + # tar xvf "$layer" -C ansible/files/postgres --strip-components 1 + # done + + # - name: Build AMI + # run: | + # GIT_SHA=${{github.sha}} + # packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=ami-test" -var "force-deregister=true" -var 'ami_regions=["ap-southeast-1"]' amazon-arm64.pkr.hcl - - name: Run aio integration tests - run: | - pip3 install docker pytest pytest-testinfra - pytest -vv testinfra/test_all_in_one.py + # - name: Cleanup resources on build cancellation + # if: ${{ cancelled() }} + # run: | + # aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -I {} aws ec2 terminate-instances --instance-ids {}