From 343ec0df231700e9ef64001be0e5d7c9c8a146e7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 22 Aug 2024 12:08:02 -0400 Subject: [PATCH 1/6] docs: updating for packaging and package updating --- nix/docs/adding-new-package.md | 160 +++++++++++++++++++++++++++++++++ nix/docs/update-extension.md | 75 +++------------- 2 files changed, 170 insertions(+), 65 deletions(-) create mode 100644 nix/docs/adding-new-package.md diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md new file mode 100644 index 000000000..5c725f479 --- /dev/null +++ b/nix/docs/adding-new-package.md @@ -0,0 +1,160 @@ +# Adding a new extension package + + +## Pre-packaging steps +1. Make sure you have nix installed [Nix installer](https://github.com/DeterminateSystems/nix-installer) +2. Create a branch off of `develop` + + +## C/C++ postgres extensions + +If you are creating a C/C++ extension, the pattern found in for instance https://github.com/supabase/postgres/blob/develop/nix/ext/pgvector.nix will work well. + +``` +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgvector"; + version = "0.7.4"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "pgvector"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Open-source vector similarity search for Postgres"; + homepage = "https://github.com/${src.owner}/${src.repo}"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} +``` + +This uses `stdenv.mkDerivation` which is a general nix builder for C and C++ projects (and others). It can auto detect the Makefile, and attempt to use it. ***It's a good practice to not have steps in the Makefile of your project that try to deal with OS specific system paths, or make calls out to the internet, as Nix cannot use these steps to build your project.*** + +Your build should produce all of the sql and control files needed for the install phase. + +1. Once you have created this file, you can add it to `nix/exts/.nix` and edit `flake.nix` and add it to the `ourExtensions` list. +2. `git add .` as nix uses git to track changes +3. In your package file, temporarily empty the `hash = "sha256<...>=";` to `hash = "";` and save and `git add https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix` +4. Run `nix build .#psql_15/exts/` to try to trigger a build, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run `nix build .#psql_15/exts/`. +5. Add any needed migrations into the `supabase/postgres` migrations directory. +6. You can then run tests locally to verify that the update of the package succeeded. +7. Now it's ready for PR review! + +## Extensions written in Rust that use `buildPgrxExtension` builder + +Extensions like: + +* https://github.com/supabase/postgres/blob/develop/nix/ext/wrappers/default.nix +* https://github.com/supabase/postgres/blob/develop/nix/ext/pg_graphql.nix +* https://github.com/supabase/postgres/blob/develop/nix/ext/pg_jsonschema.nix + +Are written in Rust, built with `cargo`, and need to use https://github.com/pgcentralfoundation/pgrx to build the extension. + +We in turn have a special nix package `builder` which is sourced from `nixpkgs` and called `buildPgrxExtension` + +A simple example is found in `pg_jsonschema` + + +``` +{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension_0_11_3, cargo }: + +buildPgrxExtension_0_11_3 rec { + pname = "pg_jsonschema"; + version = "0.3.1"; + inherit postgresql; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v${version}"; + hash = "sha256-YdKpOEiDIz60xE7C+EzpYjBcH0HabnDbtZl23CYls6g="; + }; + + nativeBuildInputs = [ cargo ]; + buildInputs = [ postgresql ]; + # update the following array when the pg_jsonschema version is updated + # required to ensure that extensions update scripts from previous versions are generated + + previousVersions = ["0.3.0" "0.2.0" "0.1.4" "0.1.4" "0.1.2" "0.1.1" "0.1.0"]; + CARGO="${cargo}/bin/cargo"; + env = lib.optionalAttrs stdenv.isDarwin { + POSTGRES_LIB = "${postgresql}/lib"; + RUSTFLAGS = "-C link-arg=-undefined -C link-arg=dynamic_lookup"; + }; + cargoHash = "sha256-VcS+efMDppofuFW2zNrhhsbC28By3lYekDFquHPta2g="; + + # FIXME (aseipp): testsuite tries to write files into /nix/store; we'll have + # to fix this a bit later. + doCheck = false; + + preBuild = '' + echo "Processing git tags..." + echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt + ''; + + postInstall = '' + echo "Creating SQL files for previous versions..." + current_version="${version}" + sql_file="$out/share/postgresql/extension/pg_jsonschema--$current_version.sql" + + if [ -f "$sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/pg_jsonschema--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$sql_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $sql_file not found" + fi + rm git_tags.txt + ''; + + + meta = with lib; { + description = "JSON Schema Validation for PostgreSQL"; + homepage = "https://github.com/supabase/${pname}"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} +``` + +Here we have built support in our overlay to specify and pin the version of `buildPgrxExtension` to a specific version (in this case `buildPgrxExtension_0_11_3`). This is currently the only version we can support, but this can be extended in our overlay https://github.com/supabase/postgres/blob/develop/nix/overlays/cargo-pgrx-0-11-3.nix to support other versions. + +A few things about `buildPgrxExtension_x`: + +* It doesn't support `buildPhase`, `installPhase` and those are implemented directly in the builder already +* It mostly just allows `cargo build` to do it's thing, but you may need to set env vars for the build process as seen above +* It caclulates a special `cargoHash` that will be generated after the first in `src` is generated, when running `nix build .#psql_15/exts/` to build the extension + + +## Post Nix derivation release steps + + +1. You can add and run tests as described in https://github.com/supabase/postgres/blob/develop/nix/docs/adding-tests.md +2. You may need to add tests to our testdatabase.yml gh action workflow as well. +3. You can add the package and name and version to `ansible/vars.yml` it is not necessary to add the sha256 hash here, as the package is already built and cached in our release process before these vars are ever run. +4. to check that all your files will land in the overall build correctly, you can run `nix profile install .#psql_15/bin` on your machine, and check in `~/.nix-profile/bin, ~/.nix-profile/lib, ~/.nix-profile/share/postgresql/*` and you should see your lib, .control and sql files there. +5. You can also run `nix run .#start-server 15` and in a new terminal window run `nix run .#star-client-and-migrate 15` and try to `CREATE EXTENSION ` and work with it there +6. Check that your extension works with the `pg_upgrade` process (TODO documentation forthcoming) +7. Now you are ready to PR the extension +8. From here, the release process should typically take care of the rest. \ No newline at end of file diff --git a/nix/docs/update-extension.md b/nix/docs/update-extension.md index 63efc895f..f96a4b00a 100644 --- a/nix/docs/update-extension.md +++ b/nix/docs/update-extension.md @@ -1,69 +1,14 @@ -Extensions are managed in two places: upstream, and this repository. Therefore -there are two update strategies available. -## Updating extensions in this repository +# Update an existing nix extension -Assuming that you run `nix develop` to get a development shell, there is a tool -named `nix-update` that is available: -``` -austin@GANON:~/work/nix-postgres$ which nix-update -/nix/store/2jyq6h0ln3f5vlgz2had80l2crdkjmdy-nix-update-0.19.2/bin/nix-update -``` +1. Create a branch off of `develop` +2. For instance, if we were updating https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix we would: + 1. change the `version = "2.2.1";` to whatever our git tag release version is that we want to update to + 2. temporarily empty the `hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE=";` to `hash = "";` and save https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix and `git add https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix` + 3. run `nix build .#psql_15/exts/supautils` or the name of the extension to update, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run nix build .#psql_15/exts/supautils. + 4. Add any needed migrations into the `supabase/postgres` migrations directory + 5. You can then run tests locally to verify that the update of the package succeeded. + 6. Now it's ready for PR review. + -Run something like this to update the extension `pg_foobar`: - -```bash -nix-update --flake psql_15/exts/pg_foobar -git commit -asm "pg_foobar: update to latest release" -``` - -It doesn't matter if you use `psql_15` or `psql_16` here, because `nix-update` -will look at the _file_ that extension is defined in, in order to update the -source code. - -## Updating extensions upstream - -You can use the same tool, `nix-update`, to do this. If you're sitting in the -root of the nixpkgs repository, try this: - -``` -nix run nixpkgs#nix-update -- postgresqlPackages.pg_foobar -git commit -asm "pg_foobar: update to latest release" -``` - -Because the tool may not be in your shell by default, we use `nix run` to run it -for us. - -The full list of available names to substitute for `pg_foobar` is available in -the file -[pkgs/servers/sql/postgresql/packages.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/sql/postgresql/packages.nix) - -### Updating the Nixpkgs snapshot - -Now that your change is merged upstream, you need to update the version of -`nixpkgs` used in this repository: - -- Check the `nixpkgs-unstable` branch: - https://github.com/nixos/nixpkgs/tree/nixpkgs-unstable -- Wait until your commit is fast-forwarded and part of that branch -- Run `nix flake update` - -## Release tags versus latest trunk - -By default, `nix-update` will update an expression to the latest tagged release. -No extra arguments are necessary. You can specify an exact release tag using the -`--version=` flag. Using the syntax `--version=branch` means "update to the -latest version on the default branch." - - -## Other notes - -See issue [#5](https://github.com/supabase/nix-postgres/issues/5) for more -information about extension management. From 90d07d8ed4737c4de39a4cd82180c7d4fec534b2 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 23 Aug 2024 05:28:23 -0400 Subject: [PATCH 2/6] Update nix/docs/adding-new-package.md Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> --- nix/docs/adding-new-package.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 5c725f479..928af7eec 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -48,7 +48,7 @@ This uses `stdenv.mkDerivation` which is a general nix builder for C and C++ pro Your build should produce all of the sql and control files needed for the install phase. -1. Once you have created this file, you can add it to `nix/exts/.nix` and edit `flake.nix` and add it to the `ourExtensions` list. +1. Once you have created this file, you can add it to `nix/ext/.nix` and edit `flake.nix` and add it to the `ourExtensions` list. 2. `git add .` as nix uses git to track changes 3. In your package file, temporarily empty the `hash = "sha256<...>=";` to `hash = "";` and save and `git add https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix` 4. Run `nix build .#psql_15/exts/` to try to trigger a build, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run `nix build .#psql_15/exts/`. From c1eb33eccf5476c0453b281a36a245bdfaf2549e Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 23 Aug 2024 05:29:12 -0400 Subject: [PATCH 3/6] Update nix/docs/adding-new-package.md Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> --- nix/docs/adding-new-package.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 928af7eec..c09c802f0 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -50,7 +50,7 @@ Your build should produce all of the sql and control files needed for the instal 1. Once you have created this file, you can add it to `nix/ext/.nix` and edit `flake.nix` and add it to the `ourExtensions` list. 2. `git add .` as nix uses git to track changes -3. In your package file, temporarily empty the `hash = "sha256<...>=";` to `hash = "";` and save and `git add https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix` +3. In your package file, temporarily empty the `hash = "sha256<...>=";` to `hash = "";` and save and `git add .` 4. Run `nix build .#psql_15/exts/` to try to trigger a build, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run `nix build .#psql_15/exts/`. 5. Add any needed migrations into the `supabase/postgres` migrations directory. 6. You can then run tests locally to verify that the update of the package succeeded. From 7d8d5bbe56eb1752b47bd304483aec0c54fecc09 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 23 Aug 2024 05:29:40 -0400 Subject: [PATCH 4/6] Update nix/docs/adding-new-package.md Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> --- nix/docs/adding-new-package.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index c09c802f0..9f07667d5 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -151,7 +151,7 @@ A few things about `buildPgrxExtension_x`: 1. You can add and run tests as described in https://github.com/supabase/postgres/blob/develop/nix/docs/adding-tests.md -2. You may need to add tests to our testdatabase.yml gh action workflow as well. +2. You may need to add tests to our test.yml gh action workflow as well. 3. You can add the package and name and version to `ansible/vars.yml` it is not necessary to add the sha256 hash here, as the package is already built and cached in our release process before these vars are ever run. 4. to check that all your files will land in the overall build correctly, you can run `nix profile install .#psql_15/bin` on your machine, and check in `~/.nix-profile/bin, ~/.nix-profile/lib, ~/.nix-profile/share/postgresql/*` and you should see your lib, .control and sql files there. 5. You can also run `nix run .#start-server 15` and in a new terminal window run `nix run .#star-client-and-migrate 15` and try to `CREATE EXTENSION ` and work with it there From 7dd2705f0fb7cf908737302d9968e6ecf0fdc5a5 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 23 Aug 2024 05:30:02 -0400 Subject: [PATCH 5/6] Update nix/docs/update-extension.md Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> --- nix/docs/update-extension.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docs/update-extension.md b/nix/docs/update-extension.md index f96a4b00a..7b41f9837 100644 --- a/nix/docs/update-extension.md +++ b/nix/docs/update-extension.md @@ -5,7 +5,7 @@ 1. Create a branch off of `develop` 2. For instance, if we were updating https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix we would: 1. change the `version = "2.2.1";` to whatever our git tag release version is that we want to update to - 2. temporarily empty the `hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE=";` to `hash = "";` and save https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix and `git add https://github.com/supabase/postgres/blob/develop/nix/ext/supautils.nix` + 2. temporarily empty the `hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE=";` to `hash = "";` and save `supautils.nix` and `git add .` 3. run `nix build .#psql_15/exts/supautils` or the name of the extension to update, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run nix build .#psql_15/exts/supautils. 4. Add any needed migrations into the `supabase/postgres` migrations directory 5. You can then run tests locally to verify that the update of the package succeeded. From d1cf39837af6f7fded7f4961334b6b24b895daab Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 23 Aug 2024 05:30:21 -0400 Subject: [PATCH 6/6] Update nix/docs/adding-new-package.md Co-authored-by: Steve Chavez --- nix/docs/adding-new-package.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 9f07667d5..30a85f7b5 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -8,7 +8,7 @@ ## C/C++ postgres extensions -If you are creating a C/C++ extension, the pattern found in for instance https://github.com/supabase/postgres/blob/develop/nix/ext/pgvector.nix will work well. +If you are creating a C/C++ extension, the pattern found in https://github.com/supabase/postgres/blob/develop/nix/ext/pgvector.nix will work well. ``` { lib, stdenv, fetchFromGitHub, postgresql }: