8000 publish docker images/image updates for major versions (#1372) · supabase/postgres@d61c5c4 · GitHub
[go: up one dir, main page]
Skip to content

Commit d61c5c4

Browse files
samrosesamrose
authored
publish docker images/image updates for major versions (#1372)
* feat: orioledb-17 Docker image and publish workflow * feat: refactor to build docker images from major versions skip if no Dockerfile exists * feat: gen common vars hcl file * chore: cleanup * chore: adj trigger for run in PR` * chore: trigger release all major vers * chore: trigger on any path * chore: cleanup version handling * chore: simplify name matching * chore: matrix on version and arch in build_release_image * chore: try to get json formatting correct * chore: more fixes to json parse * feat: collect all versions into file, then array * chore: try to account for gh action handling * chore: output single matrix for each version * chore: more fixes to matrix handling * chore : fix output versions * chore: matrix in all tasks * chore: refactor matrix handling * chore: fix syntax * chore: look in the right dir * chore: need to pass full tag * fix: formatting * chore: strip the prefix when passing to mirror.yml * chore: get version formatted to pass to mirror.yml * fix: typo * chore: fix vars * fix: tryo to pass the version * chore: matrix on version and arch * chore: try to account for orioledb version issue * chore: remove version * chore: reading the version info directly from vars.yml * chore: force string context * chore: fixing logic * chore: fix string handling logic * chore: use sed and grep instead * chore: revert this to working version * chore: version extraction * chore: different logic on version processing if stmnt * chore: try to avoid jq involvement in this section * chore: back to this version * chore: refactor using nushell where scripting needed * chore: use data from previous step * chore: install with snap * chore: raw output on json * chore: should be able to run on ephemeral runners * chore: incrementing changes to matrix * chore: just use special key name if present * chore: account for postgres namespace in matrix * chore: pg_version should be key/value * chrore: ubuntu latest * chore: needs arm-runner after all * chore: source nushell from nix * chore: cannot have uses and run keys on same id * chore: detect runner and adjust the way command is run * chore: formatting * chore: make sure env var can be appended in all contexts * chore: outputs instead of env * chore: fix get_publish_version * chore: handle oriole * chore: remove base64 encoding * ore: raw json * chore: ensure proper variable naming * chore: use the matrix config we already have access to * chore: handling architecture * chore directly use matrix * chore: process each version * chore: matrix_json variable * fix: mirroring the merged manifest instead of per arch image * chore: using tags from merge_manifest * chore: no longer need get_publish_version at all * chore: combine tags in outputs at the end of merge_manifest * chore: cleanup steps to unbreak yaml * chore: no from json * chore: try to build up json over iterations * chore: use the github outputs array * chore: first collect all versions into array then output json array * chore: debug mainfest output * chore: store the results data in artifacts * chore: unique upload, then download and combine * fix: re-add actions and login * chore: try quotes for name matching * chore: utilise versions from prepare for download of artifact * chore: format correctly for nushell * chore: parse matrix config directly * chore: convert table to list of strings * chore:rm redundant * chore: add debug * chore: from json * chore: download with pattern * chore: use list operations * chore: include and debug * chore: extract version * chore: set up for merge to develop * chore: add permission to proper dir * chore: remove old files * chore: newline
1 parent 30dbf4a commit d61c5c4

File tree

3 files changed

+280
-42
lines changed

3 files changed

+280
-42
lines changed

‎.github/workflows/dockerhub-release-matrix.yml

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
name: Release all major versions on Dockerhub
2+
3+
on:
4+
push:
5+
branches:
6+
- develop
7+
- release/*
8+
paths:
9+
- ".github/workflows/dockerhub-release-matrix.yml"
10+
workflow_dispatch:
11+
12+
jobs:
13+
prepare:
14+
runs-on: ubuntu-latest
15+
outputs:
16+
matrix_config: ${{ steps.set-matrix.outputs.matrix_config }}
17+
steps:
18+
- uses: DeterminateSystems/nix-installer-action@main
19+
- name: Checkout Repo
20+
uses: actions/checkout@v3
21+
- name: Generate build matrix
22+
id: set-matrix
23+
run: |
24+
nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major)
25+
let matrix = ($versions | each { |ver|
26+
let version = ($ver | str trim)
27+
let dockerfile = $"Dockerfile-($version)"
28+
if ($dockerfile | path exists) {
29+
{
30+
version: $version,
31+
dockerfile: $dockerfile
32+
}
33+
} else {
34+
null
35+
}
36+
} | compact)
37+
38+
let matrix_config = {
39+
include: $matrix
40+
}
41+
42+
$"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT'
43+
build:
44+
needs: prepare
45+
strategy:
46+
matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }}
47+
runs-on: ubuntu-latest
48+
outputs:
49+
build_args: ${{ steps.args.outputs.result }}
50+
steps:
51+
- uses: actions/checkout@v3
52+
- uses: DeterminateSystems/nix-installer-action@main
53+
- name: Set PostgreSQL version environment variable
54+
run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV
55+
56+
- id: args
57+
run: |
58+
nix run nixpkgs#nushell -- -c '
59+
open ansible/vars.yml
60+
| items { |key value| {name: $key, item: $value} }
61+
| where { |it| ($it.item | describe) == "string" }
62+
| each { |it| $"($it.name)=($it.item)" }
63+
| str join "\n"
64+
| save --append $env.GITHUB_OUTPUT
65+
'
66+
build_release_image:
67+
needs: [prepare, build]
68+
strategy:
69+
matrix:
70+
postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
71+
arch: [amd64, arm64]
72+
runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }}
73+
timeout-minutes: 180
74+
steps:
75+
- uses: actions/checkout@v3
76+
- uses: DeterminateSystems/nix-installer-action@main
77+
- run: docker context create builders
78+
- uses: docker/setup-buildx-action@v3
79+
with:
80+
endpoint: builders
81+
- uses: docker/login-action@v2
82+
with:
83+
username: ${{ secrets.DOCKER_USERNAME }}
84+
password: ${{ secrets.DOCKER_PASSWORD }}
85+
- name: Get image tag
86+
id: image
87+
run: |
88+
if [[ "${{ matrix.arch }}" == "arm64" ]]; then
89+
pg_version=$(sudo nix run nixpkgs#nushell -- -c '
90+
let version = "${{ matrix.postgres.version }}"
91+
let release_key = if ($version | str contains "orioledb") {
92+
$"postgresorioledb-17"
93+
} else {
94+
$"postgres($version)"
95+
}
96+
open ansible/vars.yml | get postgres_release | get $release_key | str trim
97+
')
98+
echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
99+
else
100+
pg_version=$(nix run nixpkgs#nushell -- -c '
101+
let version = "${{ matrix.postgres.version }}"
102+
let release_key = if ($version | str contains "orioledb") {
103+
$"postgresorioledb-17"
104+
} else {
105+
$"postgres($version)"
106+
}
107+
open ansible/vars.yml | get postgres_release | get $release_key | str trim
108+
')
109+
echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
110+
fi
111+
- id: build
112+
uses: docker/build-push-action@v5
113+
with:
114+
push: true
115+
build-args: |
116+
${{ needs.build.outputs.build_args }}
117+
target: production
118+
tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}
119+
platforms: linux/${{ matrix.arch }}
120+
cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
121+
cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
122+
file: ${{ matrix.postgres.dockerfile }}
123+
merge_manifest:
124+
needs: [prepare, build, build_release_image]
125+
strategy:
126+
matrix:
127+
include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
128+
runs-on: ubuntu-latest
129+
steps:
130+
- uses: actions/checkout@v3
131+
- uses: DeterminateSystems/nix-installer-action@main
132+
- uses: docker/setup-buildx-action@v3
133+
- uses: docker/login-action@v2
134+
with:
135+
username: ${{ secrets.DOCKER_USERNAME }}
136+
password: ${{ secrets.DOCKER_PASSWORD }}
137+
- name: Get image tag
138+
id: get_version
139+
run: |
140+
nix run nixpkgs#nushell -- -c '
141+
let version = "${{ matrix.version }}"
142+
let release_key = if ($version | str contains "orioledb") {
143+
$"postgresorioledb-17"
144+
} else {
145+
$"postgres($version)"
146+
}
147+
let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim)
148+
$"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT
149+
'
150+
- name: Output version
151+
id: output_version
152+
run: |
153+
echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT
154+
- name: Collect versions
155+
id: collect_versions
156+
run: |
157+
echo "${{ steps.output_version.outputs.result }}" >> results.txt # Append results
158+
- name: Upload Results Artifact
159+
uses: actions/upload-artifact@v3
160+
with:
161+
name: merge_results-${{ matrix.version }}
162+
path: results.txt
163+
if-no-files-found: warn
164+
- name: Merge multi-arch manifests
165+
run: |
166+
docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \
167+
${{ steps.get_version.outputs.pg_version }}_amd64 \
168+
${{ steps.get_version.outputs.pg_version }}_arm64
169+
combine_results:
170+
needs: [prepare, merge_manifest]
171+
runs-on: ubuntu-latest
172+
steps:
173+
- uses: actions/checkout@v3
174+
- uses: DeterminateSystems/nix-installer-action@main
175+
176+
- name: Debug Input from Prepare
177+
run: |
178+
echo "Raw matrix_config output:"
179+
echo "${{ needs.prepare.outputs.matrix_config }}"
180+
- name: Get Versions from Matrix Config
181+
id: get_versions
182+
run: |
183+
nix run nixpkgs#nushell -- -c '
184+
# Parse the matrix configuration directly
185+
let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json)
186+
187+
# Get versions directly from include array
188+
let versions = ($matrix_config.include | get version)
189+
190+
echo "Versions: $versions"
191+
192+
# Convert the versions to a comma-separated string
193+
let versions_str = ($versions | str join ",")
194+
$"versions=$versions_str" | save --append $env.GITHUB_ENV
195+
'
196+
- name: Download Results Artifacts
197+
uses: actions/download-artifact@v3
198+
with:
199+
pattern: merge_results-*
200+
- name: Combine Results
201+
id: combine
202+
run: |
203+
nix run nixpkgs#nushell -- -c '
204+
# Get all results files and process them in one go
205+
let files = (ls **/results.txt | get name)
206+
echo $"Found files: ($files)"
207+
208+
let matrix = {
209+
include: (
210+
$files
211+
| each { |file| open $file } # Open each file
212+
| each { |content| $content | lines } # Split into lines
213+
| flatten # Flatten the nested lists
214+
| where { |line| $line != "" } # Filter empty lines
215+
| each { |line|
216+
# Extract just the version part after the last colon
217+
let version = ($line | parse "supabase/postgres:{version}" | get version.0)
218+
{version: $version}
219+
}
220+
)
221+
}
222+
223+
let json_output = ($matrix | to json -r) # -r for raw output
224+
echo $"Debug output: ($json_output)"
225+
226+
$"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT
227+
'
228+
- name: Debug Combined Results
229+
run: |
230+
echo "Combined Results: '${{ steps.combine.outputs.matrix }}'"
231+
outputs:
232+
matrix: ${{ steps.combine.outputs.matrix }}
233+
publish:
234+
needs: combine_results
235+
strategy:
236+
matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }}
237+
uses: ./.github/workflows/mirror.yml
238+
with:
239+
version: ${{ matrix.version }}
240+
secrets: inherit

‎Dockerfile-15

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -41,26 +41,14 @@ ARG wal_g_release=2.0.1
4141

4242
FROM ubuntu:focal as base
4343

44-
45-
ENV DEBIAN_FRONTEND=noninteractive \
46-
DEBCONF_NONINTERACTIVE_SEEN=true \
47-
TZ=Etc/UTC
48-
49-
# Pre-configure tzdata before any installations
50-
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \
51-
echo $TZ > /etc/timezone && \
52-
apt-get update && \
53-
apt-get install -y --no-install-recommends tzdata && \
54-
apt-get install -y \
44+
RUN apt update -y && apt install -y \
5545
curl \
5646
gnupg \
5747
lsb-release \
5848
software-properties-common \
5949
wget \
6050
sudo \
61-
git \
62-
&& apt clean && \
63-
rm -rf /var/lib/apt/lists/*
51+
&& apt clean
6452

6553

6654
RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres
@@ -87,11 +75,9 @@ WORKDIR /
8775
RUN mkdir -p /usr/lib/postgresql/bin \
8876
/usr/lib/postgresql/share/postgresql \
8977
/usr/share/postgresql \
90-
# /usr/lib/postgresql/share/postgresql/contrib \
91-
#/usr/lib/postgresql/share/postgresql/timezonesets \
92-
#/usr/lib/postgresql/share/postgresql/tsearch_data \
93-
# /usr/lib/postgresql/share/postgresql/extension \
78+
/var/lib/postgresql \
9479
&& chown -R postgres:postgres /usr/lib/postgresql \
80+
&& chown -R postgres:postgres /var/lib/postgresql \
9581
&& chown -R postgres:postgres /usr/share/postgresql
9682

9783
# Create symbolic links
@@ -114,6 +100,12 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
114100
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
115101

116102

103+
RUN apt-get update && \
104+
apt-get install -y --no-install-recommends tzdata
105+
106+
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
107+
dpkg-reconfigure --frontend noninteractive tzdata
108+
117109
RUN apt-get update && \
118110
apt-get install -y --no-install-recommends \
119111
build-essential \

0 commit comments

Comments
 (0)
0