From a3af158f7b9e09b56a7ba738641dd0988a4bb56f Mon Sep 17 00:00:00 2001 From: samrose Date: Sat, 16 Mar 2024 10:29:59 +0100 Subject: [PATCH 01/15] feat: integrating a parallel nix-based build of supabase psql, exts, wrappers includes experimental integration of orioledb and patched psql 16 for orioledb --- .gitignore | 3 + flake.lock | 129 +++ flake.nix | 576 +++++++++++++ nix/docker/init.sh.in | 5 + nix/docs/README.md | 8 + nix/docs/adding-tests.md | 65 ++ nix/docs/build-postgres.md | 180 ++++ nix/docs/docker.md | 14 + nix/docs/migration-tests.md | 50 ++ nix/docs/new-major-postgres.md | 34 + nix/docs/nix-overlays.md | 34 + nix/docs/receipt-files.md | 155 ++++ nix/docs/references.md | 31 + nix/docs/start-client-server.md | 93 ++ nix/docs/start-here.md | 70 ++ nix/docs/update-extension.md | 69 ++ nix/docs/use-direnv.md | 150 ++++ ...001-build-Allow-using-V8-from-system.patch | 46 + nix/ext/citus.nix | 37 + nix/ext/hypopg.nix | 31 + nix/ext/orioledb.nix | 32 + nix/ext/pg_graphql.nix | 29 + nix/ext/pg_hashids.nix | 31 + nix/ext/pg_jsonschema.nix | 28 + nix/ext/pg_net.nix | 31 + nix/ext/pg_plan_filter.nix | 30 + nix/ext/pg_stat_monitor.nix | 51 ++ nix/ext/pg_tle.nix | 34 + nix/ext/pgsodium.nix | 31 + nix/ext/pgsql-http.nix | 31 + nix/ext/pgvector.nix | 31 + nix/ext/plv8.nix | 147 ++++ nix/ext/supautils.nix | 29 + nix/ext/vault.nix | 30 + nix/ext/wrappers/default.nix | 61 ++ nix/init.sh | 20 + nix/overlays/cargo-pgrx.nix | 27 + nix/overlays/gdal-small.nix | 14 + nix/overlays/psql_16-oriole.nix | 21 + nix/tests/migrations/data.sql | 21 + nix/tests/postgresql.conf.in | 797 ++++++++++++++++++ nix/tests/prime.sql | 23 + nix/tests/smoke/0000-hello-world.sql | 10 + nix/tests/smoke/0001-pg_graphql.sql | 59 ++ nix/tests/smoke/0002-supautils.sql | 17 + nix/tests/smoke/0003-pgsodium-vault.sql | 40 + nix/tests/util/pgsodium_getkey.sh | 4 + nix/tools/README.md | 2 + nix/tools/migrate-tool.sh.in | 123 +++ nix/tools/run-client.sh.in | 29 + nix/tools/run-replica.sh.in | 43 + nix/tools/run-server.sh.in | 45 + 52 files changed, 3701 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/docker/init.sh.in create mode 100644 nix/docs/README.md create mode 100644 nix/docs/adding-tests.md create mode 100644 nix/docs/build-postgres.md create mode 100644 nix/docs/docker.md create mode 100644 nix/docs/migration-tests.md create mode 100644 nix/docs/new-major-postgres.md create mode 100644 nix/docs/nix-overlays.md create mode 100644 nix/docs/receipt-files.md create mode 100644 nix/docs/references.md create mode 100644 nix/docs/start-client-server.md create mode 100644 nix/docs/start-here.md create mode 100644 nix/docs/update-extension.md create mode 100644 nix/docs/use-direnv.md create mode 100644 nix/ext/0001-build-Allow-using-V8-from-system.patch create mode 100644 nix/ext/citus.nix create mode 100644 nix/ext/hypopg.nix create mode 100644 nix/ext/orioledb.nix create mode 100644 nix/ext/pg_graphql.nix create mode 100644 nix/ext/pg_hashids.nix create mode 100644 nix/ext/pg_jsonschema.nix create mode 100644 nix/ext/pg_net.nix create mode 100644 nix/ext/pg_plan_filter.nix create mode 100644 nix/ext/pg_stat_monitor.nix create mode 100644 nix/ext/pg_tle.nix create mode 100644 nix/ext/pgsodium.nix create mode 100644 nix/ext/pgsql-http.nix create mode 100644 nix/ext/pgvector.nix create mode 100644 nix/ext/plv8.nix create mode 100644 nix/ext/supautils.nix create mode 100644 nix/ext/vault.nix create mode 100644 nix/ext/wrappers/default.nix create mode 100755 nix/init.sh create mode 100644 nix/overlays/cargo-pgrx.nix create mode 100644 nix/overlays/gdal-small.nix create mode 100644 nix/overlays/psql_16-oriole.nix create mode 100644 nix/tests/migrations/data.sql create mode 100644 nix/tests/postgresql.conf.in create mode 100644 nix/tests/prime.sql create mode 100644 nix/tests/smoke/0000-hello-world.sql create mode 100644 nix/tests/smoke/0001-pg_graphql.sql create mode 100644 nix/tests/smoke/0002-supautils.sql create mode 100644 nix/tests/smoke/0003-pgsodium-vault.sql create mode 100755 nix/tests/util/pgsodium_getkey.sh create mode 100644 nix/tools/README.md create mode 100644 nix/tools/migrate-tool.sh.in create mode 100644 nix/tools/run-client.sh.in create mode 100644 nix/tools/run-replica.sh.in create mode 100644 nix/tools/run-server.sh.in diff --git a/.gitignore b/.gitignore index 1b2009550..fdaf213c0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ ansible/image-manifest*.json __pycache__/ *.py[cod] *$py.class + +#nix related +result* diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..59e9002bb --- /dev/null +++ b/flake.lock @@ -0,0 +1,129 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1708764364, + "narHash": "sha256-+pOtDvmuVTg0Gi58hKDUyrNla5NbyUvt3Xs3gLR0Fws=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "c891f90d2e3c48a6b33466c96e4851e0fc0cf455", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1697269602, + "narHash": "sha256-dSzV7Ud+JH4DPVD9od53EgDrxUVQOcSj4KGjggCDVJI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9cb540e9c1910d74a7e10736277f6eb9dff51c81", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1709386671, + "narHash": "sha256-VPqfBnIJ+cfa78pd4Y5Cr6sOWVW8GYHRVucxJGmRf8Q=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fa9a51752f1b5de583ad5213eb621be071806663", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..d4c29fc20 --- /dev/null +++ b/flake.nix @@ -0,0 +1,576 @@ +{ + description = "Prototype tooling for deploying PostgreSQL"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + nix2container.url = "github:nlewo/nix2container"; + }; + + nixConfig = { + extra-substituters = [ + "https://nix-postgres-artifacts.s3.amazonaws.com" + ]; + + extra-trusted-public-keys = [ + "nix-cache.supabase.com-1:ZfEc7Qb7SN+qOTJGMtCz54rnVQ1W2ZI2ROCSSD6YQYc=" + ]; + }; + + outputs = { self, nixpkgs, flake-utils, nix2container }: + let + gitRev = "vcs=${self.shortRev or "dirty"}+${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}"; + + ourSystems = with flake-utils.lib; [ + system.x86_64-linux + system.aarch64-linux + ]; + in + flake-utils.lib.eachSystem ourSystems (system: + let + pgsqlDefaultPort = "5435"; + pgsqlSuperuser = "postgres"; + nix2img = nix2container.packages.${system}.nix2container; + + # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which + # we can use to build our own images; it is the common name to refer to + # a copy of nixpkgs which contains all its packages. + # it also serves as a base for importing the orioldb/postgres overlay to + #build the orioledb postgres patched version of postgresql16 + oriole_pkgs = import nixpkgs { + inherit system; + overlays = [ + # NOTE (aseipp): add any needed overlays here. in theory we could + # pull them from the overlays/ directory automatically, but we don't + # want to have an arbitrary order, since it might matter. being + # explicit is better. + (import ./nix/overlays/cargo-pgrx.nix) + (import ./nix/overlays/gdal-small.nix) + (import ./nix/overlays/psql_16-oriole.nix) + + ]; + }; + #This variable works the same as 'oriole_pkgs' but builds using the upstream + #nixpkgs builds of postgresql 15 and 16 + the overlays listed below + pkgs = import nixpkgs { + inherit system; + overlays = [ + # NOTE (aseipp): add any needed overlays here. in theory we could + # pull them from the overlays/ directory automatically, but we don't + # want to have an arbitrary order, since it might matter. being + # explicit is better. + (import ./nix/overlays/cargo-pgrx.nix) + (import ./nix/overlays/gdal-small.nix) + + ]; + }; + + + # FIXME (aseipp): pg_prove is yet another perl program that needs + # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we + # can remove this wrapper. + pg_prove = pkgs.runCommand "pg_prove" + { + nativeBuildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + for x in pg_prove pg_tapgen; do + makeWrapper "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}/bin/$x" "$out/bin/$x" \ + --set LOCALE_ARCHIVE "${pkgs.glibcLocales}/lib/locale/locale-archive" + done + ''; + + + # Our list of PostgreSQL extensions which come from upstream Nixpkgs. + # These are maintained upstream and can easily be used here just by + # listing their name. Anytime the version of nixpkgs is upgraded, these + # may also bring in new versions of the extensions. + psqlExtensions = [ + "postgis" + "pgrouting" + "pgtap" + "pg_cron" + "pgaudit" + "pgjwt" + "plpgsql_check" + "pg_safeupdate" + "wal2json" + /* pljava */ + "rum" + "pg_repack" + "pgroonga" + "timescaledb" + ]; + + #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue + # with building timescaledb with the orioledb patched version of postgresql + orioledbPsqlExtensions = [ + "postgis" + "pgrouting" + "pgtap" + "pg_cron" + "pgaudit" + "pgjwt" + "plpgsql_check" + "pg_safeupdate" + "wal2json" + /* pljava */ + "rum" + "pg_repack" + "pgroonga" + /*"timescaledb"*/ + ]; + + # Custom extensions that exist in our repository. These aren't upstream + # either because nobody has done the work, maintaining them here is + # easier and more expedient, or because they may not be suitable, or are + # too niche/one-off. + # + # Ideally, most of these should have copies upstream for third party + # use, but even if they did, keeping our own copies means that we can + # rollout new versions of these critical things easier without having to + # go through the upstream release engineering process. + ourExtensions = [ + # see comment in ./nix/ext/citus.nix + # for why citus is deactivated + #./nix/ext/citus.nix + ./nix/ext/pgsql-http.nix + ./nix/ext/pg_plan_filter.nix + ./nix/ext/pg_net.nix + ./nix/ext/pg_hashids.nix + ./nix/ext/pgsodium.nix + ./nix/ext/pg_graphql.nix + ./nix/ext/pg_stat_monitor.nix + ./nix/ext/pg_jsonschema.nix + ./nix/ext/pgvector.nix + ./nix/ext/vault.nix + ./nix/ext/hypopg.nix + ./nix/ext/pg_tle.nix + ./nix/ext/wrappers/default.nix + ./nix/ext/supautils.nix + ./nix/ext/plv8.nix + ]; + + #Where we import and build the orioledb extension, we add on our custom extensions + # plus the orioledb option + orioledbExtension = ourExtensions ++ [./nix/ext/orioledb.nix ]; + + #this var is a convenience setting to import the orioledb patched version of postgresql + postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + + # Create a 'receipt' file for a given postgresql package. This is a way + # of adding a bit of metadata to the package, which can be used by other + # tools to inspect what the contents of the install are: the PSQL + # version, the installed extensions, et cetera. + # + # This takes three arguments: + # - pgbin: the postgresql package we are building on top of + # - upstreamExts: the list of extensions from upstream nixpkgs. This is + # not a list of packages, but an attrset containing extension names + # mapped to versions. + # - ourExts: the list of extensions from upstream nixpkgs. This is not + # a list of packages, but an attrset containing extension names + # mapped to versions. + # + # The output is a package containing the receipt.json file, which can be + # merged with the PostgreSQL installation using 'symlinkJoin'. + makeReceipt = pgbin: upstreamExts: ourExts: pkgs.writeTextFile { + name = "receipt"; + destination = "/receipt.json"; + text = builtins.toJSON { + revision = gitRev; + psql-version = pgbin.version; + nixpkgs = { + revision = nixpkgs.rev; + extensions = upstreamExts; + }; + extensions = ourExts; + + # NOTE (aseipp): this field can be used to do cache busting (e.g. + # force a rebuild of the psql packages) but also to helpfully inform + # tools what version of the schema is being used, for forwards and + # backwards compatibility + receipt-version = "1"; + }; + }; + + makeOurOrioleDbPostgresPkgs = version: patchedPostgres: + let postgresql = patchedPostgres; + in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; + + makeOurPostgresPkgs = version: + let postgresql = pkgs."postgresql_${version}"; + in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions; + + # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. + makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: + (builtins.listToAttrs (map + (drv: + { name = drv.pname; value = drv; } + ) + (makeOurOrioleDbPostgresPkgs version patchedPostgres))) + // { recurseForDerivations = true; }; + + # Create an attrset that contains all the extensions included in a server. + makeOurPostgresPkgsSet = version: + (builtins.listToAttrs (map + (drv: + { name = drv.pname; value = drv; } + ) + (makeOurPostgresPkgs version))) + // { recurseForDerivations = true; }; + + + # Create a binary distribution of PostgreSQL, given a version. + # + # NOTE: The version here does NOT refer to the exact PostgreSQL version; + # it refers to the *major number only*, which is used to select the + # correct version of the package from nixpkgs. This is because we want + # to be able to do so in an open ended way. As an example, the version + # "15" passed in will use the nixpkgs package "postgresql_15" as the + # basis for building extensions, etc. + makePostgresBin = version: + let + postgresql = pkgs."postgresql_${version}"; + upstreamExts = map + (ext: { + name = postgresql.pkgs."${ext}".pname; + version = postgresql.pkgs."${ext}".version; + }) + psqlExtensions; + ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurPostgresPkgs version); + + pgbin = postgresql.withPackages (ps: + (map (ext: ps."${ext}") psqlExtensions) ++ (makeOurPostgresPkgs version) + ); + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + }; + + makeOrioleDbPostgresBin = version: patchedPostgres: + let + postgresql = patchedPostgres; + upstreamExts = map + (ext: { + name = postgresql.pkgs."${ext}".pname; + version = postgresql.pkgs."${ext}".version; + }) + orioledbPsqlExtensions; + ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); + + pgbin = postgresql.withPackages (ps: + (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) + ); + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + }; + + # Make a Docker Image from a given PostgreSQL version and binary package. + # updated to use https://github.com/nlewo/nix2container (samrose) + makePostgresDocker = version: binPackage: + let + initScript = pkgs.runCommand "docker-init.sh" { } '' + mkdir -p $out/bin + substitute ${./nix/docker/init.sh.in} $out/bin/init.sh \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' + + chmod +x $out/bin/init.sh + ''; + + postgresqlConfig = pkgs.runCommand "postgresql.conf" { } '' + mkdir -p $out/etc/ + substitute ${./nix/tests/postgresql.conf.in} $out/etc/postgresql.conf \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" + ''; + + l = pkgs.lib // builtins; + + user = "postgres"; + group = "postgres"; + uid = "1001"; + gid = "1001"; + + mkUser = pkgs.runCommand "mkUser" { } '' + mkdir -p $out/etc/pam.d + + echo "${user}:x:${uid}:${gid}::" > $out/etc/passwd + echo "${user}:!x:::::::" > $out/etc/shadow + + echo "${group}:x:${gid}:" > $out/etc/group + echo "${group}:x::" > $out/etc/gshadow + + cat > $out/etc/pam.d/other <logfile 2>&1 & + sleep 2 + + createdb -h localhost testing + + psql -h localhost -d testing -Xaf ${./nix/tests/prime.sql} + pg_prove -h localhost -d testing ${sqlTests}/*.sql + + pkill postgres + mv logfile $out + echo ${pgpkg} + ''; + + in + rec { + # The list of all packages that can be built with 'nix build'. The list + # of names that can be used can be shown with 'nix flake show' + packages = flake-utils.lib.flattenTree basePackages // { + # Any extra packages we might want to include in our package + # set can go here. + inherit (pkgs) + # NOTE: comes from our cargo-pgrx.nix overlay + cargo-pgrx_0_11_2; + + }; + + # The list of exported 'checks' that are run with every run of 'nix + # flake check'. This is run in the CI system, as well. + checks = { + psql_15 = makeCheckHarness basePackages.psql_15.bin; + psql_16 = makeCheckHarness basePackages.psql_16.bin; + psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; + }; + + # Apps is a list of names of things that can be executed with 'nix run'; + # these are distinct from the things that can be built with 'nix build', + # so they need to be listed here too. + apps = + let + mkApp = attrName: binName: { + type = "app"; + program = "${basePackages."${attrName}"}/bin/${binName}"; + }; + in + { + start-server = mkApp "start-server" "start-postgres-server"; + start-client = mkApp "start-client" "start-postgres-client"; + start-replica = mkApp "start-replica" "start-postgres-replica"; + migration-test = mkApp "migrate-tool" "migrate-postgres"; + }; + + # 'devShells.default' lists the set of packages that are included in the + # ambient $PATH environment when you run 'nix develop'. This is useful + # for development and puts many convenient devtools instantly within + # reach. + devShells.default = pkgs.mkShell { + packages = with pkgs; [ + coreutils + just + nix-update + pg_prove + shellcheck + + basePackages.start-server + basePackages.start-client + basePackages.start-replica + basePackages.migrate-tool + ]; + shellHook = '' + export HISTFILE=.history + ''; + }; + } + ); +} diff --git a/nix/docker/init.sh.in b/nix/docker/init.sh.in new file mode 100644 index 000000000..360d44001 --- /dev/null +++ b/nix/docker/init.sh.in @@ -0,0 +1,5 @@ +#!/bin/bash +# shellcheck shell=bash +/bin/initdb --locale=C -D /data/postgresql +ln -s /etc/postgresql.conf /data/postgresql/postgresql.conf +/bin/postgres -p @PGSQL_DEFAULT_PORT@ -D /data/postgresql diff --git a/nix/docs/README.md b/nix/docs/README.md new file mode 100644 index 000000000..400632984 --- /dev/null +++ b/nix/docs/README.md @@ -0,0 +1,8 @@ +# Documentation + +This directory contains most of the "runbooks" and documentation on how to use +this repository. + +You probably want to start with the [starting guide](./start-here.md). Then, +learn how to play with `postgres` in the [build guide](./build-postgres.md). +After that, you can probe around a bit. diff --git a/nix/docs/adding-tests.md b/nix/docs/adding-tests.md new file mode 100644 index 000000000..f417873a4 --- /dev/null +++ b/nix/docs/adding-tests.md @@ -0,0 +1,65 @@ +There are basically two types of tests you can add: + +- pgTAP based tests, and +- Migration tests. + +In both cases, a number of extensions may be installed into the database for +use; you can see those in both [postgresql.conf.in](../tests/postgresql.conf.in) +and [prime.sql](../tests/prime.sql) (extensions may be enabled in either place.) + +## pgTAP tests + +These are super easy: simply add `.sql` files to the +[tests/smoke](./../tests/smoke/) directory, then: + +``` +nix flake check -L +``` + +(`-L` prints logs to stderrr, for more details see `man nix`) + +These files are run using `pg_prove`; they pretty much behave exactly like how +you expect; you can read +[the pgTAP documentation](https://pgtap.org/documentation.html) for more. + +For a good example of a pgTAP test as a pull request, check out +[pull request #4](https://github.com/supabase/nix-postgres/pull/4/files). + +## Re-running tests + +`nix flake check` gets its results cached, so if you do it again the tests won't rerun. If you change a file then it will run again. + +If you want to force rerun without modifying a file, you can do: + +``` +nix build .#checks.x86_64-linux.psql_14 --rebuild +nix build .#checks.x86_64-linux.psql_15 --rebuild +``` + +Limitation: currently there's no way to rerun all the tests, so you have to specify the check attribute. + +To get the correct attribute (`#checks.x86_64-linux.psql_15` above), you can do `nix flake show`. This will show a tree with all the output attributes. + +## Migration tests + +> **NOTE**: Currently, migration tests _do not happen in CI_. They can only be +> run manually. + +Migration tests are pretty simple in the sense they follow a very simple +principle: + +- You put data in the database +- Run the migration procedure +- It should probably not fail + +Step 1 and 2 are easy, and for various reasons (e.g. mistakes from upstream +extension authors), step 3 isn't guaranteed, so that's what the whole idea is +designed to test. + +To add data into the database, modify the +[data.sql](../tests/migrations/data.sql) script and add whatever you want into +it. This script gets loaded into the old version of the database at startup, and +it's expected that the new version of the database can handle it. + +To run the `migration-test` tool, check out the documentation on +[migration-tests](./migration-tests.md). diff --git a/nix/docs/build-postgres.md b/nix/docs/build-postgres.md new file mode 100644 index 000000000..791bb98f6 --- /dev/null +++ b/nix/docs/build-postgres.md @@ -0,0 +1,180 @@ +# 01 — Use this repository + +Let's clone this repo: + +```bash +git clone https://github.com/supabase/nix-postgres $HOME/tmp-nix-postgres +cd $HOME/tmp-nix-postgres +``` + +## Hashes for everyone + +But how do we build stuff within it? With `nix build`, of course! For example, +the following command will, when completed, create a symlink named `result` that +points to a path which contains an entire PostgreSQL 15 installation — +extensions and all: + +``` +nix build .#psql_15/bin +``` + +``` +$ readlink result +/nix/store/ybf48481x033649mgdzk5dyaqv9dppzx-postgresql-and-plugins-15.3 +``` + +``` +$ ls result +bin include lib share +``` + +``` +$ ll result/bin/ +total 9928 +dr-xr-xr-x 2 root root 4096 Dec 31 1969 ./ +dr-xr-xr-x 5 root root 4096 Dec 31 1969 ../ +lrwxrwxrwx 1 root root 79 Dec 31 1969 .initdb-wrapped -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/.initdb-wrapped* +-r-xr-xr-x 1 root root 9829624 Dec 31 1969 .postgres-wrapped* +lrwxrwxrwx 1 root root 73 Dec 31 1969 clusterdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/clusterdb* +lrwxrwxrwx 1 root root 72 Dec 31 1969 createdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/createdb* +lrwxrwxrwx 1 root root 74 Dec 31 1969 createuser -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/createuser* +lrwxrwxrwx 1 root root 70 Dec 31 1969 dropdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/dropdb* +lrwxrwxrwx 1 root root 72 Dec 31 1969 dropuser -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/dropuser* +lrwxrwxrwx 1 root root 68 Dec 31 1969 ecpg -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/ecpg* +lrwxrwxrwx 1 root root 70 Dec 31 1969 initdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/initdb* +lrwxrwxrwx 1 root root 72 Dec 31 1969 oid2name -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/oid2name* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_amcheck -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_amcheck* +lrwxrwxrwx 1 root root 81 Dec 31 1969 pg_archivecleanup -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_archivecleanup* +lrwxrwxrwx 1 root root 77 Dec 31 1969 pg_basebackup -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_basebackup* +lrwxrwxrwx 1 root root 76 Dec 31 1969 pg_checksums -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_checksums* +-r-xr-xr-x 1 root root 53432 Dec 31 1969 pg_config* +lrwxrwxrwx 1 root root 78 Dec 31 1969 pg_controldata -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_controldata* +-r-xr-xr-x 1 root root 82712 Dec 31 1969 pg_ctl* +lrwxrwxrwx 1 root root 71 Dec 31 1969 pg_dump -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_dump* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_dumpall -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_dumpall* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_isready -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_isready* +lrwxrwxrwx 1 root root 77 Dec 31 1969 pg_receivewal -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_receivewal* +lrwxrwxrwx 1 root root 78 Dec 31 1969 pg_recvlogical -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_recvlogical* +lrwxrwxrwx 1 root root 73 Dec 31 1969 pg_repack -> /nix/store/bi9i5ns4cqxk235qz3srs9p4x1qfxfna-pg_repack-1.4.8/bin/pg_repack* +lrwxrwxrwx 1 root root 75 Dec 31 1969 pg_resetwal -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_resetwal* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_restore -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_restore* +lrwxrwxrwx 1 root root 73 Dec 31 1969 pg_rewind -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_rewind* +lrwxrwxrwx 1 root root 77 Dec 31 1969 pg_test_fsync -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_test_fsync* +lrwxrwxrwx 1 root root 78 Dec 31 1969 pg_test_timing -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_test_timing* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_upgrade -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_upgrade* +lrwxrwxrwx 1 root root 79 Dec 31 1969 pg_verifybackup -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_verifybackup* +lrwxrwxrwx 1 root root 74 Dec 31 1969 pg_waldump -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pg_waldump* +lrwxrwxrwx 1 root root 71 Dec 31 1969 pgbench -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/pgbench* +lrwxrwxrwx 1 root root 71 Dec 31 1969 pgsql2shp -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgsql2shp* +lrwxrwxrwx 1 root root 77 Dec 31 1969 pgsql2shp-3.3.3 -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgsql2shp-3.3.3* +lrwxrwxrwx 1 root root 75 Dec 31 1969 pgtopo_export -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgtopo_export* +lrwxrwxrwx 1 root root 81 Dec 31 1969 pgtopo_export-3.3.3 -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgtopo_export-3.3.3* +lrwxrwxrwx 1 root root 75 Dec 31 1969 pgtopo_import -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgtopo_import* +lrwxrwxrwx 1 root root 81 Dec 31 1969 pgtopo_import-3.3.3 -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/pgtopo_import-3.3.3* +-r-xr-xr-x 1 root root 286 Dec 31 1969 postgres* +lrwxrwxrwx 1 root root 74 Dec 31 1969 postmaster -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/postmaster* +lrwxrwxrwx 1 root root 68 Dec 31 1969 psql -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/psql* +lrwxrwxrwx 1 root root 74 Dec 31 1969 raster2pgsql -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/raster2pgsql* +lrwxrwxrwx 1 root root 80 Dec 31 1969 raster2pgsql-3.3.3 -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/raster2pgsql-3.3.3* +lrwxrwxrwx 1 root root 73 Dec 31 1969 reindexdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/reindexdb* +lrwxrwxrwx 1 root root 71 Dec 31 1969 shp2pgsql -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/shp2pgsql* +lrwxrwxrwx 1 root root 77 Dec 31 1969 shp2pgsql-3.3.3 -> /nix/store/4wwzd3c136g6j7aqva2gyiqgwy784qjv-postgis-3.3.3/bin/shp2pgsql-3.3.3* +lrwxrwxrwx 1 root root 72 Dec 31 1969 vacuumdb -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/vacuumdb* +lrwxrwxrwx 1 root root 72 Dec 31 1969 vacuumlo -> /nix/store/kdjdxnyhpwpvb11da8s99ylqilspcmzl-postgresql-15.3/bin/vacuumlo* +``` + +As we can see, these files all point to paths under `/nix/store`. We're actually +looking at a "farm" of symlinks to various paths, but collectively they form an +entire installation directory we can reuse as much as we want. + +The path +`/nix/store/ybf48481x033649mgdzk5dyaqv9dppzx-postgresql-and-plugins-15.3` +ultimately is a cryptographically hashed, unique name for our installation of +PostgreSQL with those plugins. This hash includes _everything_ used to build it, +so even a single change anywhere to any extension or version would result in a +_new_ hash. + +The ability to refer to a piece of data by its hash, by some notion of +_content_, is a very powerful primitive, as we'll see later. + +## Build a different version: v14 + +What if we wanted PostgreSQL 14 and plugins? Just replace `_15` with `_14`: + +``` +nix build .#psql_14/bin +``` + +You're done: + +``` +$ readlink result +/nix/store/p7ziflx0000s28bfb213jsghrczknkc4-postgresql-and-plugins-14.8 +``` + +## Do it all at once: using `just` + +But remembering that long name is tedious. (Don't worry, there's a method to +query the names, and we will go over it later.) What if we just wanted to build +something quickly? + +There's a great tool for this. It's called +[**Just**](https://github.com/casey/just), and all it does is make it easy to +run commands. Let's use that. + +Luckily, there's a `justfile` that can be used to build multiple things. Here's +the `build-all` rule in our [`justfile`](../justfile). It should be obvious if +you're familiar with `make` and `Makefile`s: + +``` +build-all: + nix build .#psql_14/bin .#psql_14/docker + nix build .#psql_15/bin .#psql_15/docker +``` + +So this actually builds _four_ things: v14 and v15 of Postgres, both as a binary +distribution _and_ as a Docker image. That's handy, instead of writing it all +out. So we run `just build-all`, and... + +``` +$ just build-all +Command 'just' not found, did you mean: +... +``` + +Ouch. So we have to install `just` using Cargo or something else first. Or do +we? + +## Using `nix develop` + +Nope! Here's how we can add `just` to our `$PATH`, transparently in a subshell: + +``` +$ which just + +$ echo $$ +766420 + +$ nix develop + +$ which just +/nix/store/1di6nb4qsv86907l3xarw4llzdss2g89-just-1.14.0/bin/just + +$ echo $$ +996868 + +$ just build-all +... + +$ exit + +$ echo $$ +766420 +``` + +Done! As you can see, `nix develop .` will just drop you in a subshell with +tools you need _ready to go instantly_. That's all you need to do! And once that +shell goes away, `just` will be removed from your `$PATH` as well. + +There's an even easier way to do this +[that is completely transparent to you, as well](./use-direnv.md). diff --git a/nix/docs/docker.md b/nix/docs/docker.md new file mode 100644 index 000000000..198c18d49 --- /dev/null +++ b/nix/docs/docker.md @@ -0,0 +1,14 @@ +Docker images are pushed to `ghcr.io` on every commit. Try the following: + +``` +docker run --rm -it ghcr.io/supabase/nix-postgres-15:latest +``` + +Every Docker image that is built on every push is given a tag that exactly +corresponds to a Git commit in the repository — for example commit +[d3e0c39d34e1bb4d37e058175a7bc376620f6868](https://github.com/supabase/nix-postgres/commit/d3e0c39d34e1bb4d37e058175a7bc376620f6868) +in this repository has a tag in the container registry which can be used to pull +exactly that version. + +This just starts the server. Client container images are not provided; you can +use `nix run` for that, as outlined [here](./start-client-server.md). diff --git a/nix/docs/migration-tests.md b/nix/docs/migration-tests.md new file mode 100644 index 000000000..d04bfeb90 --- /dev/null +++ b/nix/docs/migration-tests.md @@ -0,0 +1,50 @@ +Migration tests are run similar to running the client and server; see +[more on that here](./start-client-server.md). + +Instead, you use the following format to specify the upgrade: + +``` +nix run .#migration-test [pg_dumpall|pg_upgrade] +``` + +The arguments are: + +- The version to upgrade from +- The version to upgrade to +- The upgrade mechanism: either `pg_dumpall` or `pg_upgrade` + +## Specifying the version + +The versions for upgrading can be one of two forms: + +- A major version number, e.g. `14` or `15` +- A path to `/nix/store`, which points to _any_ version of PostgreSQL, as long + as it has the "expected" layout and is a postgresql install. + +## Always use the latest version of the migration tool + +Unlike the method for starting the client or server, you probably always want to +use the latest version of the `migration-test` tool from the repository. This is +because it can ensure forwards and backwards compatibility if necessary. + +## Upgrading between arbitrary `/nix/store` versions + +If you want to test migrations from arbitrary versions built by the repository, +you can combine `nix build` and `nix run` to do so. You can use the syntax from +the runbook on [running the server & client](./start-client-server.md) to refer +to arbitrary git revisions. + +For example, if you updated an extension in this repository, and you want to +test a migration from PostgreSQL 14 to PostgreSQL 14 + (updated extension), +using `pg_upgrade` — simply record the two git commits you want to +compare, and you could do something like the following: + +``` +OLD_GIT_VERSION=... +NEW_GIT_VERSION=... + +nix run github:supabase/nix-postgres#migration-test \ + $(nix build "github:supabase/nix-postgres/$OLD_GIT_VERSION#psql_14/bin") \ + $(nix build "github:supabase/nix-postgres/$NEW_GIT_VERSION#psql_14/bin") \ + pg_upgrade +``` diff --git a/nix/docs/new-major-postgres.md b/nix/docs/new-major-postgres.md new file mode 100644 index 000000000..d9d929273 --- /dev/null +++ b/nix/docs/new-major-postgres.md @@ -0,0 +1,34 @@ +PostgreSQL versions are managed in upstream nixpkgs. + +See this example PR to add a new version of PostgreSQL; this version is for 16 +beta3, but any version is roughly the same. In short, you need to: + +- Add a new version and hash +- Possibly patch the source code for minor refactorings + - In this example, an old patch had to be rewritten because a function was + split into two different functions; the patch is functionally equivalent but + textually different +- Add the changes to `all-packages.nix` +- Integrate inside the CI and get code review +- Run `nix flake update` to get a new version, once it's ready + +https://github.com/NixOS/nixpkgs/pull/249030 + +## Adding the major version to this repository + +It isn't well abstracted, unfortunately. In short: look for the strings `14` and +`15` under `flake.nix` and `tools/`. More specifically: + +- Add `psql_XX` to `basePackages` in `flake.nix` +- Ditto with `checks` in `flake.nix` +- Modify the tools under `tools/` to understand the new major version +- Make sure the CI is integrated under the GitHub Actions. + +The third step and fourth steps are the most annoying, really. The first two are +easy and by that point you can run `nix flake check` in order to test the build, +at least. + +## Other notes + +See also issue [#6](https://github.com/supabase/nix-postgres/issues/6), which +would make it possible to define PostgreSQL versions inside this repository. diff --git a/nix/docs/nix-overlays.md b/nix/docs/nix-overlays.md new file mode 100644 index 000000000..eced66b98 --- /dev/null +++ b/nix/docs/nix-overlays.md @@ -0,0 +1,34 @@ +Overlays are a feature of Nixpkgs that allow you to: + +- Add new packages with new names to the namespace _without_ modifying upstream + - For example, if there is a package `foobar`, you might add `foobar-1_2_3` to + add a specific version for backwards compatibility +- Globally override _existing_ package names, in terms of other packages. + - For example, if you want to globally override a package to enable a + disabled-by-default feature. + +First, you need to define a file for the overlay under +[overlays/](../overlays/), and then import it in `flake.nix`. There is an +example pull request in +[#14](https://github.com/supabase/nix-postgres/issues/14) for this; an overlay +typically looks like this: + +``` +final: prev: { + gdal = prev.gdalMinimal; +} +``` + +This says "globally override `gdal` with a different version, named +`gdalMinimal`". In this case `gdalMinimal` is a build with less features +enabled. + +The most important part is that there is an equation of the form `lhs = rhs;` +— if the `lhs` refers to an existing name, it's overwritten. If it refers +to a new name, it's introduced. Overwriting an existing name acts as if you +changed the files upstream: so the above example _globally_ overrides GDAL for +anything that depends on it. + +The names `final` and `prev` are used to refer to packages in terms of other +overlays. For more information about this, see the +[NixOS Wiki Page for Overlays](https://nixos.wiki/wiki/Overlays). diff --git a/nix/docs/receipt-files.md b/nix/docs/receipt-files.md new file mode 100644 index 000000000..978f8a9f4 --- /dev/null +++ b/nix/docs/receipt-files.md @@ -0,0 +1,155 @@ +Every time you run `nix build` on this repository to build PostgreSQL, the +installation directory comes with a _receipt_ file that tells you what's inside +of it. Primarily, this tells you: + +- The version of PostgreSQL, +- The installed extensions, and +- The version of nixpkgs. + +The intent of the receipt file is to provide a mechanism for tooling to +understand installation directories and provide things like upgrade paths or +upgrade mechanisms. + +## Example receipt + +For example: + +``` +nix build .#psql_15/bin +``` + +``` +austin@GANON:~/work/nix-postgres$ nix build .#psql_15/bin +austin@GANON:~/work/nix-postgres$ ls result +bin include lib receipt.json share +``` + +The receipt is in JSON format, under `receipt.json`. Here's an example of what +it would look like: + +```json +{ + "extensions": [ + { + "name": "pgsql-http", + "version": "1.5.0" + }, + { + "name": "pg_plan_filter", + "version": "unstable-2021-09-23" + }, + { + "name": "pg_net", + "version": "0.7.2" + }, + { + "name": "pg_hashids", + "version": "unstable-2022-09-17" + }, + { + "name": "pgsodium", + "version": "3.1.8" + }, + { + "name": "pg_graphql", + "version": "unstable-2023-08-01" + }, + { + "name": "pg_stat_monitor", + "version": "1.0.1" + }, + { + "name": "pg_jsonschema", + "version": "unstable-2023-07-23" + }, + { + "name": "vault", + "version": "0.2.9" + }, + { + "name": "hypopg", + "version": "1.3.1" + }, + { + "name": "pg_tle", + "version": "1.0.4" + }, + { + "name": "supabase-wrappers", + "version": "unstable-2023-07-31" + }, + { + "name": "supautils", + "version": "1.7.3" + } + ], + "nixpkgs": { + "extensions": [ + { + "name": "postgis", + "version": "3.3.3" + }, + { + "name": "pgrouting", + "version": "3.5.0" + }, + { + "name": "pgtap", + "version": "1.2.0" + }, + { + "name": "pg_cron", + "version": "1.5.2" + }, + { + "name": "pgaudit", + "version": "1.7.0" + }, + { + "name": "pgjwt", + "version": "unstable-2021-11-13" + }, + { + "name": "plpgsql_check", + "version": "2.3.4" + }, + { + "name": "pg-safeupdate", + "version": "1.4" + }, + { + "name": "timescaledb", + "version": "2.11.1" + }, + { + "name": "wal2json", + "version": "2.5" + }, + { + "name": "plv8", + "version": "3.1.5" + }, + { + "name": "rum", + "version": "1.3.13" + }, + { + "name": "pgvector", + "version": "0.4.4" + }, + { + "name": "pg_repack", + "version": "1.4.8" + }, + { + "name": "pgroonga", + "version": "3.0.8" + } + ], + "revision": "750fc50bfd132a44972aa15bb21937ae26303bc4" + }, + "psql-version": "15.3", + "receipt-version": "1", + "revision": "vcs=d250647+20230814" +} +``` diff --git a/nix/docs/references.md b/nix/docs/references.md new file mode 100644 index 000000000..fe5b791e2 --- /dev/null +++ b/nix/docs/references.md @@ -0,0 +1,31 @@ +Nix references and other useful tools: + +- **Zero to Nix**: Start here to get your feet wet with how Nix works, and how + to use Nixpkgs: https://zero-to-nix.com/ +- `nix-installer`: My recommended way to install Nix + - https://github.com/DeterminateSystems/nix-installer +- Nix manual https://nixos.org/manual/nix/stable/ + - Useful primarily for option and command references +- Flake schema reference https://nixos.wiki/wiki/Flakes + - Useful to know what `flake.nix` is referring to +- Example pull requests for this repo: + - Adding smoke tests for an extension: + https://github.com/supabase/nix-postgres/pull/2 + - Extension smoke tests, part 2: + https://github.com/supabase/nix-postgres/pull/3 + - Adding an extension and a smoke test at once: + https://github.com/supabase/nix-postgres/pull/4/files + - Updating an extension to trunk: + https://github.com/supabase/nix-postgres/pull/7 + - Updating an extension to the latest release: + https://github.com/supabase/nix-postgres/pull/9 +- Contributing to [nixpkgs](https://github.com/nixos/nixpkgs) + - Adding a PGRX-powered extension: + https://github.com/NixOS/nixpkgs/pull/246803 + - Adding a normal extension: https://github.com/NixOS/nixpkgs/pull/249000 + - Adding new PostgreSQL versions: https://github.com/NixOS/nixpkgs/pull/249030 +- NixOS Discourse: https://discourse.nixos.org/ + - Useful for community feedback, guidance, and help +- `nix-update`: https://github.com/Mic92/nix-update + - Used in this repository to help update extensions +- pgTAP for testing: https://pgtap.org/documentation.html diff --git a/nix/docs/start-client-server.md b/nix/docs/start-client-server.md new file mode 100644 index 000000000..2bde53327 --- /dev/null +++ b/nix/docs/start-client-server.md @@ -0,0 +1,93 @@ +## Running the server + +If you want to run a postgres server, just do this from the root of the +repository: + +``` +nix run .#start-server 14 +``` + +Replace the `14` with a `15`, and you'll be using a different version. Optionally you can specify a second argument for the port. + +You likely have a running postgres, so to not cause a conflict, this uses port 5435 by default. + +Actually, you don't even need the repository. You can do this from arbitrary +directories, if the left-hand side of the hash character (`.` in this case) is a +valid "flake reference": + +``` +# from any arbitrary directory +nix run github:supabase/nix-postgres#start-server 14 +``` + +### Arbitrary versions at arbitrary git revisions + +Let's say you want to use a PostgreSQL build from a specific version of the +repository. You can change the syntax of the above to use _any_ version of the +repository, at any time, by adding the commit hash after the repository name: + +``` +# use postgresql 15 build at commit a9989d4800dd6038827afed27456f19ba4b2ae0a +nix run github:supabase/nix-postgres/a9989d4800dd6038827afed27456f19ba4b2ae0a#start-server 15 +``` + +## Running the client + +All of the same rules apply, but try using `start-client` on the right-hand side +of the hash character, instead. For example: + +``` +nix run github:supabase/nix-postgres#start-server 14 & +sleep 5 +nix run github:supabase/nix-postgres#start-client 15 +``` + +## Running a server replica + +To start a replica you can use the `start-postgres-replica` command. + +- first argument: the master version +- second argument: the master port +- third argument: the replica server port + +First start a server and a couple of replicas: + +``` +$ start-postgres-server 15 5435 + +$ start-postgres-replica 15 5439 + +$ start-postgres-replica 15 5440 +``` + +Now check the master server: + +``` +$ start-postgres-client 15 5435 +``` + +```sql +SELECT client_addr, state +FROM pg_stat_replication; + client_addr | state +-------------+----------- + ::1 | streaming + ::1 | streaming +(2 rows) + +create table items as select x::int from generate_series(1,100) x; +``` + +And a replica: + +``` +$ start-postgres-client 15 5439 +``` + +```sql +select count(*) from items; + count +------- + 100 +(1 row) +``` diff --git a/nix/docs/start-here.md b/nix/docs/start-here.md new file mode 100644 index 000000000..acc315830 --- /dev/null +++ b/nix/docs/start-here.md @@ -0,0 +1,70 @@ +Let's go ahead and install Nix. To do that, we'll use the +**[nix-installer tool]** by Determinate Systems. This works on many platforms, +but most importantly it works on **aarch64 Linux** and **x86_64 Linux**. Use the +following command in your shell, **it should work on any Linux distro of your +choice**: + +[nix-installer tool]: https://github.com/DeterminateSystems/nix-installer + +```bash +curl \ + --proto '=https' --tlsv1.2 \ + -sSf -L https://install.determinate.systems/nix \ +| sh -s -- install +``` + +After you do this, **you must log in and log back out of your desktop +environment** to get a new login session. This is so that your shell can have +the Nix tools installed on `$PATH` and so that your user shell can see some +extra settings. + +You should now be able to do something like the following; try running these +same commands on your machine: + +``` +$ nix --version +nix (Nix) 2.16.1 +``` + +``` +$ nix run nixpkgs#nix-info -- -m + - system: `"x86_64-linux"` + - host os: `Linux 5.15.90.1-microsoft-standard-WSL2, Ubuntu, 22.04.2 LTS (Jammy Jellyfish), nobuild` + - multi-user?: `yes` + - sandbox: `yes` + - version: `nix-env (Nix) 2.16.1` + - channels(root): `"nixpkgs"` + - nixpkgs: `/nix/var/nix/profiles/per-user/root/channels/nixpkgs` +``` + +If the above worked, you're now cooking with gas! + +> _**NOTE**_: While there is an upstream tool to install Nix, written in Bash, +> we use the Determinate Systems installer — which will hopefully replace the +> original — because it's faster, and takes care of several extra edge cases +> that the original one couldn't handle, and makes several changes to the +> default installed configuration to make things more user friendly. Determinate +> Systems is staffed by many long-time Nix contributors and the creator of Nix, +> and is trustworthy. + +## Do some fun stuff + +One of the best things about Nix that requires _very little_ knowledge of it is +that it lets you install the latest and greatest versions of many tools _on any +Linux distribution_. We'll explain more about that later on. But just as a few +examples: + +- **Q**: I want the latest version of Deno. Can we get that? +- **A**: `nix profile install nixpkgs#deno`, and you're done! + + + +- **Q**: What about HTTPie? A nice Python application? +- **A**: Same idea: `nix profile install nixpkgs#httpie` + + + +- **Q**: What about my favorite Rust applications, like ripgrep and bat? +- **A.1**: `nix profile install nixpkgs#ripgrep` +- **A.2**: `nix profile install nixpkgs#bat` +- **A.3**: And yes, you also have exa, fd, hyperfine, and more! diff --git a/nix/docs/update-extension.md b/nix/docs/update-extension.md new file mode 100644 index 000000000..f318a6e36 --- /dev/null +++ b/nix/docs/update-extension.md @@ -0,0 +1,69 @@ +Extensions are managed in two places: upstream, and this repository. Therefore +there are two update strategies available. + +## Updating extensions in this repository + +Assuming that you run `nix develop` to get a development shell, there is a tool +named `nix-update` that is available: + +``` +austin@GANON:~/work/nix-postgres$ which nix-update +/nix/store/2jyq6h0ln3f5vlgz2had80l2crdkjmdy-nix-update-0.19.2/bin/nix-update +``` + +Run something like this to update the extension `pg_foobar`: + +```bash +nix-update --flake psql_15/exts/pg_foobar +git commit -asm "pg_foobar: update to latest release" +``` + +It doesn't matter if you use `psql_14` or `psql_15` here, because `nix-update` +will look at the _file_ that extension is defined in, in order to update the +source code. + +## Updating extensions upstream + +You can use the same tool, `nix-update`, to do this. If you're sitting in the +root of the nixpkgs repository, try this: + +``` +nix run nixpkgs#nix-update -- postgresqlPackages.pg_foobar +git commit -asm "pg_foobar: update to latest release" +``` + +Because the tool may not be in your shell by default, we use `nix run` to run it +for us. + +The full list of available names to substitute for `pg_foobar` is available in +the file +[pkgs/servers/sql/postgresql/packages.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/sql/postgresql/packages.nix) + +### Updating the Nixpkgs snapshot + +Now that your change is merged upstream, you need to update the version of +`nixpkgs` used in this repository: + +- Check the `nixpkgs-unstable` branch: + https://github.com/nixos/nixpkgs/tree/nixpkgs-unstable +- Wait until your commit is fast-forwarded and part of that branch +- Run `nix flake update` + +## Release tags versus latest trunk + +By default, `nix-update` will update an expression to the latest tagged release. +No extra arguments are necessary. You can specify an exact release tag using the +`--version=` flag. Using the syntax `--version=branch` means "update to the +latest version on the default branch." + +## Example PRs + +- https://github.com/supabase/nix-postgres/pull/9 updates `pg_net` to the latest + release +- https://github.com/supabase/nix-postgres/pull/7 updates `pg_hashids` to the + latest `trunk` tip + +## Other notes + +See issue [#5](https://github.com/supabase/nix-postgres/issues/5) for more +information about extension management. diff --git a/nix/docs/use-direnv.md b/nix/docs/use-direnv.md new file mode 100644 index 000000000..bad00ea64 --- /dev/null +++ b/nix/docs/use-direnv.md @@ -0,0 +1,150 @@ +Have you ever used a tool like `pip`'s `bin/activate` script, or `rbenv`? These +tools populate your shell environment with the right tools and scripts and +dependencies (e.g. `PYTHONPATH`) to run your software. + +What if I told you there was a magical tool that worked like that, and could do +it for arbitrary languages and tools? + +That tool is called **[direnv](https://direnv.net)**. + +## Install direnv and use it in your shell + +First, install `direnv`: + +``` +$ nix profile install nixpkgs#direnv +``` + +``` +$ which direnv +/home/austin/.nix-profile/bin/direnv +``` + +Now, you need to activate it in your shell by hooking into it. If you're using +**Bash**, try putting this in your `.bashrc` and starting up a new interactive +shell: + +``` +eval "$(direnv hook bash)" +``` + +Not using bash? Check the +[direnv hook documentation](https://direnv.net/docs/hook.html) for more. + +## Set up `nix-postgres` + +Let's go back to the `nix-postgres` source code. + +``` +cd $HOME/tmp-nix-postgres +``` + +Now, normally, direnv is going to look for a file called `.envrc` and load that +if it exists. But to be polite, we don't do that by default; we keep a file +named `.envrc.recommended` in the repository instead, and encourage people to do +this: + +``` +echo "source_env .envrc.recommended" >> .envrc +``` + +All this says is "Load the code from `.envrc.recommended` directly", just like a +normal bash script using `source`. The idea of this pattern is to allow users to +have their own customized `.envrc` and piggyback on the committed code for +utility — and `.envrc` is `.gitignore`'d, so you can put e.g. secret +tokens inside without fear of committing them. + +Run the above command, and then... + +## What just happened? + +Oops, a big red error appeared? + +``` +$ echo "source_env .envrc.recommended" >> .envrc +direnv: error /home/austin/work/nix-postgres/.envrc is blocked. Run `direnv allow` to approve its content +``` + +What happened? By default, as a security measure, `direnv` _does not_ load or +execute any code from an `.envrc` file, and instead it MUST be allowed +explicitly. + +## `direnv allow` + +Our `.envrc.recommended` file will integrate with Nix directly. So run +`direnv allow`, and you'll suddenly see the following: + +``` +$ direnv allow +direnv: loading ~/work/nix-postgres/.envrc +direnv: loading ~/work/nix-postgres/.envrc.recommended +direnv: loading https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc (sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8=) +direnv: using flake +direnv: nix-direnv: renewed cache +direnv: export +AR +AS +CC +CONFIG_SHELL +CXX +DETERMINISTIC_BUILD +HOST_PATH +IN_NIX_SHELL +LD +NIX_BINTOOLS +NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_BUILD_CORES +NIX_CC +NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_CFLAGS_COMPILE +NIX_ENFORCE_NO_NATIVE +NIX_HARDENING_ENABLE +NIX_LDFLAGS +NIX_STORE +NM +OBJCOPY +OBJDUMP +PYTHONHASHSEED +PYTHONNOUSERSITE +PYTHONPATH +RANLIB +READELF +SIZE +SOURCE_DATE_EPOCH +STRINGS +STRIP +_PYTHON_HOST_PLATFORM +_PYTHON_SYSCONFIGDATA_NAME +__structuredAttrs +buildInputs +buildPhase +builder +cmakeFlags +configureFlags +depsBuildBuild +depsBuildBuildPropagated +depsBuildTarget +depsBuildTargetPropagated +depsHostHost +depsHostHostPropagated +depsTargetTarget +depsTargetTargetPropagated +doCheck +doInstallCheck +dontAddDisableDepTrack +mesonFlags +name +nativeBuildInputs +out +outputs +patches +phases +preferLocalBuild +propagatedBuildInputs +propagatedNativeBuildInputs +shell +shellHook +stdenv +strictDeps +system ~PATH ~XDG_DATA_DIRS +``` + +What just happened is that we populated the ambient shell environment with tools +specified inside of `flake.nix` — we'll cover Flakes later. But for now, +your tools are provisioned! + +## Run `just` to build the code + +Previously, we needed to run `nix develop` to get access to the `just` command. +No longer! It's now available in your shell: + +``` +$ which just +/nix/store/1di6nb4qsv86907l3xarw4llzdss2g89-just-1.14.0/bin/just +``` + +``` +$ just b +... +``` + +Viola, done! + +## Go back `$HOME` and return again + +When you `cd ~`, direnv will automatically unload the `.envrc` file for you: + +``` +$ cd ~ +direnv: unloading +``` + +Now, `just` is no longer available: + +``` +$ just build-all +just: command not found +``` + +And if you go back, it is! + +``` +$ cd $HOME/nix-postgres +direnv: loading ~/work/nix-postgres/.envrc +direnv: loading ~/work/nix-postgres/.envrc.recommended +direnv: loading https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc (sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8=) +direnv: using flake +... +``` + +``` +$ just build-all +... +``` + +## The power of `direnv` + +`direnv` with Nix is a frighteningly good development combination for many +purposes. This is its main power: you can use it to create on-demand developer +shells for any language, tool, or environment, and all you need to do is `cd` to +the right directory. + +This is the power of `direnv`: your projects always, on demand, will have the +right tools configured and available, no matter if you last worked on them a day +ago or a year ago, or it was done by your teammate, or you have a brand new +computer that you've never programmed on. diff --git a/nix/ext/0001-build-Allow-using-V8-from-system.patch b/nix/ext/0001-build-Allow-using-V8-from-system.patch new file mode 100644 index 000000000..eb5e2964f --- /dev/null +++ b/nix/ext/0001-build-Allow-using-V8-from-system.patch @@ -0,0 +1,46 @@ +diff --git a/Makefile b/Makefile +index 38879cc..6e78eeb 100644 +--- a/Makefile ++++ b/Makefile +@@ -20,6 +20,7 @@ OBJS = $(SRCS:.cc=.o) + MODULE_big = plv8-$(PLV8_VERSION) + EXTENSION = plv8 + PLV8_DATA = plv8.control plv8--$(PLV8_VERSION).sql $(wildcard upgrade/*.sql) ++USE_SYSTEM_V8 = 0 + + + # Platform detection +@@ -41,6 +42,7 @@ PGXS := $(shell $(PG_CONFIG) --pgxs) + PG_VERSION_NUM := $(shell cat `$(PG_CONFIG) --includedir-server`/pg_config*.h \ + | perl -ne 'print $$1 and exit if /PG_VERSION_NUM\s+(\d+)/') + ++ifeq ($(USE_SYSTEM_V8),0) + AUTOV8_DIR = build/v8 + AUTOV8_OUT = build/v8/out.gn/obj + AUTOV8_STATIC_LIBS = -lv8_libplatform -lv8_libbase +@@ -66,6 +68,7 @@ v8: + make -f Makefiles/Makefile.macos v8 + endif + endif ++endif + + # enable direct jsonb conversion by default + CCFLAGS += -DJSONB_DIRECT_CONVERSION +@@ -83,6 +86,7 @@ ifdef BIGINT_GRACEFUL + endif + + ++ifeq ($(USE_SYSTEM_V8),0) + # We're gonna build static link. Rip it out after include Makefile + SHLIB_LINK := $(filter-out -lv8, $(SHLIB_LINK)) + +@@ -101,6 +105,7 @@ else + SHLIB_LINK += -lrt -std=c++14 + endif + endif ++endif + + DATA = $(PLV8_DATA) + ifndef DISABLE_DIALECT +-- +2.37.3 diff --git a/nix/ext/citus.nix b/nix/ext/citus.nix new file mode 100644 index 000000000..0bb61ec31 --- /dev/null +++ b/nix/ext/citus.nix @@ -0,0 +1,37 @@ +{ lib, curl, lz4, zstd, krb5, icu, stdenv, fetchFromGitHub, postgresql }: +#FIXME we have decided deactivate this for now, as it is not used in production +# if we decide to use it later we may need to look at this documentation +# https://docs.citusdata.com/en/stable/admin_guide/upgrading_citus.html#upgrading-postgresql-version-from-15-to-16 +stdenv.mkDerivation rec { + pname = "citus"; + version = "12.1.2"; + + buildInputs = [ curl lz4 zstd krb5 icu.dev postgresql ]; + + src = fetchFromGitHub { + owner = "citusdata"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-0uYNMLAYigtGlDRvOEkQeC5i58QfXcdSVjTQwWVFX+8="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp src/backend/columnar/citus_columnar.so $out/lib + cp src/backend/columnar/citus_columnar.control $out/share/postgresql/extension + cp src/backend/columnar/build/sql/*.sql $out/share/postgresql/extension + + cp src/backend/distributed/citus.so $out/lib + cp src/backend/distributed/citus.control $out/share/postgresql/extension + cp src/backend/distributed/build/sql/*.sql $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Distributed PostgreSQL as an extension"; + homepage = "https://github.com/citusdata/${pname}"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.agpl3Plus; + }; +} diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix new file mode 100644 index 000000000..ea41c57ee --- /dev/null +++ b/nix/ext/hypopg.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "hypopg"; + version = "1.3.1"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "HypoPG"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-AIBXy+LxyHUo+1hd8gQTwaBdFiTEzKaCVc4cx5tZgME="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Hypothetical Indexes for PostgreSQL"; + homepage = "https://github.com/HypoPG/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix new file mode 100644 index 000000000..4d8c51bbd --- /dev/null +++ b/nix/ext/orioledb.nix @@ -0,0 +1,32 @@ +{ lib, stdenv, fetchFromGitHub, curl, libkrb5, postgresql, python3, openssl }: + +stdenv.mkDerivation rec { + pname = "orioledb"; + name = pname; + src = fetchFromGitHub { + owner = "orioledb"; + repo = "orioledb"; + rev = "main"; + sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY="; + }; + version = "patches16_23"; + buildInputs = [ curl libkrb5 postgresql python3 openssl ]; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23"; + installPhase = '' + runHook preInstall + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + + runHook postInstall + ''; + doCheck = true; + meta = with lib; { + description = "orioledb"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_graphql.nix b/nix/ext/pg_graphql.nix new file mode 100644 index 000000000..7ef16e060 --- /dev/null +++ b/nix/ext/pg_graphql.nix @@ -0,0 +1,29 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension }: + +buildPgrxExtension rec { + pname = "pg_graphql"; + version = "unstable-1.5.0"; + inherit postgresql; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v1.5.0"; + hash = "sha256-28ANRZyF22qF2YAxNAAkPfGOM3+xiO6IHdXsTp0CTQE="; + }; + + cargoHash = "sha256-CUiGs0m9aUeqjpdPyOSjz91cP7TT6kjJqnw7ImGnQuo="; + + # FIXME (aseipp): disable the tests since they try to install .control + # files into the wrong spot, aside from that the one main test seems + # to work, though + doCheck = false; + + meta = with lib; { + description = "GraphQL support for PostreSQL"; + homepage = "https://github.com/supabase/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix new file mode 100644 index 000000000..9d237d09c --- /dev/null +++ b/nix/ext/pg_hashids.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg_hashids"; + version = "unstable-2022-09-17"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "iCyberon"; + repo = pname; + rev = "cd0e1b31d52b394a0df64079406a14a4f7387cd6"; + hash = "sha256-Nmb7XLqQflYZfqj0yrewfb1Hl5YgEB5wfjBunPwIuOU="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Generate short unique IDs in PostgreSQL"; + homepage = "https://github.com/iCyberon/pg_hashids"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix new file mode 100644 index 000000000..df357f337 --- /dev/null +++ b/nix/ext/pg_jsonschema.nix @@ -0,0 +1,28 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension }: + +buildPgrxExtension rec { + pname = "pg_jsonschema"; + version = "unstable-v0.3.0"; + inherit postgresql; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v0.3.0"; + hash = "sha256-am6Ye+pOoAsOr9L4vJXw4iIOJ9x0VkUjqH6PdXMUZrk="; + }; + + cargoHash = "sha256-tiiWzu/mTKL5ruvWn6IxrXVhVqS4LXzjfacdFT9rbOY="; + + # FIXME (aseipp): testsuite tries to write files into /nix/store; we'll have + # to fix this a bit later. + doCheck = false; + + meta = with lib; { + description = "JSON Schema Validation for PostgreSQL"; + homepage = "https://github.com/supabase/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix new file mode 100644 index 000000000..919ef8b4a --- /dev/null +++ b/nix/ext/pg_net.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, curl, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg_net"; + version = "0.7.2"; + + buildInputs = [ curl postgresql ]; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-9Ki3fyinHTYrfckxAY0fCTlzJd9l+n7QRUV7mIWrqmc="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Async networking for Postgres"; + homepage = "https://github.com/supabase/pg_net"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix new file mode 100644 index 000000000..49d839fcc --- /dev/null +++ b/nix/ext/pg_plan_filter.nix @@ -0,0 +1,30 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg_plan_filter"; + version = "unstable-2021-09-23"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "pgexperts"; + repo = pname; + rev = "5081a7b5cb890876e67d8e7486b6a64c38c9a492"; + hash = "sha256-YNeIfmccT/DtOrwDmpYFCuV2/P6k3Zj23VWBDkOh6sw="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Filter PostgreSQL statements by execution plans"; + homepage = "https://github.com/pgexperts/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix new file mode 100644 index 000000000..35e6b3bfa --- /dev/null +++ b/nix/ext/pg_stat_monitor.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +let + # NOTE (aseipp): the 1.x series of pg_stat_monitor has some non-standard and + # weird build logic (Percona projects in general seem to have their own + # strange build harness) where it will try to pick the right .sql file to + # install into the extension dir based on the postgresql major version. for + # our purposes, we only need to support v13 and v14+, so just replicate this + # logic from the makefile and pick the right file here. + # + # this seems to all be cleaned up in version 2.0 of the extension, so ideally + # we could upgrade to it later on and nuke this. + # sqlFilename = if lib.versionOlder postgresql.version "14" + # then "pg_stat_monitor--1.0.13.sql.in" + # else "pg_stat_monitor--1.0.14.sql.in"; + +in +stdenv.mkDerivation rec { + pname = "pg_stat_monitor"; + version = "2.0.4"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "percona"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-57Ji/KltIHNf81OxT0+4JIDqydST5RKMqrybNBZochg="; + }; + + makeFlags = [ "USE_PGXS=1" ]; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp pg_stat_monitor--2.0.sql pg_stat_monitor--1.0--2.0.sql + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Query Performance Monitoring Tool for PostgreSQL"; + homepage = "https://github.com/percona/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + broken = lib.versionOlder postgresql.version "15"; + }; +} diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix new file mode 100644 index 000000000..626cd8a89 --- /dev/null +++ b/nix/ext/pg_tle.nix @@ -0,0 +1,34 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, flex }: + +stdenv.mkDerivation rec { + pname = "pg_tle"; + version = "1.0.4"; + + nativeBuildInputs = [ flex ]; + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "aws"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-W/7pLy/27VatCdzUh1NZ4K2FRMD1erfHiFV2eY2x2W0="; + }; + + makeFlags = [ "FLEX=flex" ]; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Framework for 'Trusted Language Extensions' in PostgreSQL"; + homepage = "https://github.com/aws/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix new file mode 100644 index 000000000..d97aa80c7 --- /dev/null +++ b/nix/ext/pgsodium.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgsodium"; + version = "3.1.8"; + + buildInputs = [ libsodium postgresql ]; + + src = fetchFromGitHub { + owner = "michelp"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-j5F1PPdwfQRbV8XJ8Mloi8FvZF0MTl4eyIJcBYQy1E4="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Modern cryptography for PostgreSQL"; + homepage = "https://github.com/michelp/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix new file mode 100644 index 000000000..e94e2ba9d --- /dev/null +++ b/nix/ext/pgsql-http.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, curl, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgsql-http"; + version = "1.6.0"; + + buildInputs = [ curl postgresql ]; + + src = fetchFromGitHub { + owner = "pramsey"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-CPHfx7vhWfxkXsoKTzyFuTt47BPMvzi/pi1leGcuD60="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "HTTP client for Postgres"; + homepage = "https://github.com/pramsey/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix new file mode 100644 index 000000000..996a671ff --- /dev/null +++ b/nix/ext/pgvector.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgvector"; + version = "0.5.1"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "pgvector"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-ZNzq+dATZn9LUgeOczsaadr5hwdbt9y/+sAOPIdr77U="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Open-source vector similarity search for Postgres"; + homepage = "https://github.com/${src.owner}/${src.repo}"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix new file mode 100644 index 000000000..0ff1fb4dc --- /dev/null +++ b/nix/ext/plv8.nix @@ -0,0 +1,147 @@ +{ stdenv +, lib +, fetchFromGitHub +, v8 +, perl +, postgresql + # For test +, runCommand +, coreutils +, gnugrep +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "plv8"; + # plv8 latest is https://github.com/plv8/plv8/releases/tag/v3.2.2 + # FIXME we need to increment this build toward 3.2.2 + # 3.1.7 is the highest version that can be built with pg 16 + version = "3.1.7"; + + src = fetchFromGitHub { + owner = "plv8"; + repo = "plv8"; + rev = "v${finalAttrs.version}"; + hash = "sha256-kTID3Zo3YwNZUno8kdQE7ihtiddsIAZNuBN91IKgaC4="; + }; + + patches = [ + # Allow building with system v8. + # https://github.com/plv8/plv8/pull/505 (rejected) + ./0001-build-Allow-using-V8-from-system.patch + ]; + + nativeBuildInputs = [ + perl + ]; + + buildInputs = [ + v8 + postgresql + ]; + + buildFlags = [ "all" ]; + + makeFlags = [ + # Nixpkgs build a v8 monolith instead of separate v8_libplatform. + "USE_SYSTEM_V8=1" + "SHLIB_LINK=-lv8" + "V8_OUTDIR=${v8}/lib" + ]; + + installFlags = [ + # PGXS only supports installing to postgresql prefix so we need to redirect this + "DESTDIR=${placeholder "out"}" + ]; + + # No configure script. + dontConfigure = true; + + postPatch = '' + patchShebangs ./generate_upgrade.sh + # https://github.com/plv8/plv8/pull/506 + substituteInPlace generate_upgrade.sh \ + --replace " 2.3.10 " " 2.3.10 2.3.11 2.3.12 2.3.13 2.3.14 2.3.15 " + ''; + + postInstall = '' + # Move the redirected to proper directory. + # There appear to be no references to the install directories + # so changing them does not cause issues. + mv "$out/nix/store"/*/* "$out" + rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" + ''; + + passthru = { + tests = + let + postgresqlWithSelf = postgresql.withPackages (_: [ + finalAttrs.finalPackage + ]); + in + { + smoke = runCommand "plv8-smoke-test" { } '' + export PATH=${lib.makeBinPath [ + postgresqlWithSelf + coreutils + gnugrep + ]} + db="$PWD/testdb" + initdb "$db" + postgres -k "$db" -D "$db" & + pid="$!" + + for i in $(seq 1 100); do + if psql -h "$db" -d postgres -c "" 2>/dev/null; then + break + elif ! kill -0 "$pid"; then + exit 1 + else + sleep 0.1 + fi + done + + psql -h "$db" -d postgres -c 'CREATE EXTENSION plv8; DO $$ plv8.elog(NOTICE, plv8.version); $$ LANGUAGE plv8;' 2> "$out" + grep -q "${finalAttrs.version}" "$out" + kill -0 "$pid" + ''; + + regression = stdenv.mkDerivation { + name = "plv8-regression"; + inherit (finalAttrs) src patches nativeBuildInputs buildInputs dontConfigure; + + buildPhase = '' + runHook preBuild + + # The regression tests need to be run in the order specified in the Makefile. + echo -e "include Makefile\nprint_regress_files:\n\t@echo \$(REGRESS)" > Makefile.regress + REGRESS_TESTS=$(make -f Makefile.regress print_regress_files) + + ${postgresql}/lib/pgxs/src/test/regress/pg_regress \ + --bindir='${postgresqlWithSelf}/bin' \ + --temp-instance=regress-instance \ + --dbname=contrib_regression \ + $REGRESS_TESTS + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + touch "$out" + + runHook postInstall + ''; + }; + }; + }; + + meta = with lib; { + description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; + homepage = "https://plv8.github.io/"; + maintainers = with maintainers; [ marsam ]; + platforms = [ "x86_64-linux" "aarch64-linux" ]; + license = licenses.postgresql; + broken = postgresql.jitSupport; + }; +}) diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix new file mode 100644 index 000000000..9e6d4d455 --- /dev/null +++ b/nix/ext/supautils.nix @@ -0,0 +1,29 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "supautils"; + version = "1.8.0"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-cQ294UNhlPtnqngGTVLYPMbGcqhqjFk5y6WBz6nCZhI="; + }; + + installPhase = '' + mkdir -p $out/lib + + install -D supautils.so -t $out/lib + ''; + + meta = with lib; { + description = "PostgreSQL extension for enhanced security"; + homepage = "https://github.com/supabase/${pname}"; + maintainers = with maintainers; [ steve-chavez ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix new file mode 100644 index 000000000..4e2684ab2 --- /dev/null +++ b/nix/ext/vault.nix @@ -0,0 +1,30 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "vault"; + version = "0.2.9"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-kXTngBW4K6FkZM8HvJG2Jha6OQqbejhnk7tchxy031I="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Store encrypted secrets in PostgreSQL"; + homepage = "https://github.com/supabase/${pname}"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix new file mode 100644 index 000000000..89299a2bd --- /dev/null +++ b/nix/ext/wrappers/default.nix @@ -0,0 +1,61 @@ +{ lib +, stdenv +, fetchFromGitHub +, openssl +, pkg-config +, postgresql +, buildPgrxExtension_0_11_2 +}: + +buildPgrxExtension_0_11_2 rec { + pname = "supabase-wrappers"; + version = "unstable-2024-02-26"; + inherit postgresql; + + src = fetchFromGitHub { + owner = "supabase"; + repo = "wrappers"; + #rev pinned for now to the HEAD of the main branch to achieve cargo-pgrx 0.11.2 compat + rev = "5b5c2622268c75bec834a38b2ff967f781511188"; + hash = "sha256-VwEFJD0yD+gvXCTzq9NfjCPEkh/lDQdEOPfk8LwK4z4="; + }; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl ]; + + # Needed to get openssl-sys to use pkg-config. + OPENSSL_NO_VENDOR = 1; + + cargoLock = { + lockFile = "${src}/Cargo.lock"; + outputHashes = { + "clickhouse-rs-1.0.0-alpha.1" = "sha256-0zmoUo/GLyCKDLkpBsnLAyGs1xz6cubJhn+eVqMEMaw="; + }; + }; + postPatch = "cp ${cargoLock.lockFile} Cargo.lock"; + + buildAndTestSubdir = "wrappers"; + buildFeatures = [ + "helloworld_fdw" + "bigquery_fdw" + "clickhouse_fdw" + "stripe_fdw" + "firebase_fdw" + "s3_fdw" + "airtable_fdw" + "logflare_fdw" + ]; + + # FIXME (aseipp): disable the tests since they try to install .control + # files into the wrong spot, aside from that the one main test seems + # to work, though + doCheck = false; + + meta = with lib; { + description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; + homepage = "https://github.com/supabase/wrappers"; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/init.sh b/nix/init.sh new file mode 100755 index 000000000..fc9adafcf --- /dev/null +++ b/nix/init.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# shellcheck shell=bash + +export PGUSER=postgres +export PGDATA=$PWD/postgres_data +export PGHOST=$PWD/postgres +export PGPORT=5432 +export PGPASS=postgres +export LOG_PATH=$PGHOST/LOG +export PGDATABASE=testdb +export DATABASE_URL="postgresql:///$PGDATABASE?host=$PGHOST&port=$PGPORT" +mkdir -p $PGHOST +if [ ! -d $PGDATA ]; then + echo 'Initializing postgresql database...' + initdb $PGDATA --locale=C --username $PGUSER -A md5 --pwfile=<(echo $PGPASS) --auth=trust + echo "listen_addresses='*'" >> $PGDATA/postgresql.conf + echo "unix_socket_directories='$PGHOST'" >> $PGDATA/postgresql.conf + echo "unix_socket_permissions=0700" >> $PGDATA/postgresql.conf +fi +chmod o-rwx $PGDATA diff --git a/nix/overlays/cargo-pgrx.nix b/nix/overlays/cargo-pgrx.nix new file mode 100644 index 000000000..7254d2908 --- /dev/null +++ b/nix/overlays/cargo-pgrx.nix @@ -0,0 +1,27 @@ +final: prev: { + cargo-pgrx_0_11_2 = prev.cargo-pgrx.overrideAttrs (oldAttrs: rec { + pname = "cargo-pgrx"; + version = "0.11.2"; + + src = prev.fetchCrate { + inherit version pname; + hash = "sha256-8NlpMDFaltTIA8G4JioYm8LaPJ2RGKH5o6sd6lBHmmM="; + }; + + # NOTE (aseipp): normally, we would just use 'cargoHash' here, but + # due to a fantastic interaction of APIs, we can't do that so + # easily, and have to use this incantation instead, which is + # basically the exact same thing but with 4 extra lines. see: + # + # https://discourse.nixos.org/t/is-it-possible-to-override-cargosha256-in-buildrustpackage/4393/5 + cargoDeps = oldAttrs.cargoDeps.overrideAttrs (prev.lib.const { + name = "${pname}-vendor.tar.gz"; + inherit src; + outputHash = "sha256-qU2r67qI+aWsWr3vMWHb2FItHzwSaqXDnTvRe0rf+JY="; + }); + }); + + buildPgrxExtension_0_11_2 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx_0_11_2; + }; +} diff --git a/nix/overlays/gdal-small.nix b/nix/overlays/gdal-small.nix new file mode 100644 index 000000000..18be8a526 --- /dev/null +++ b/nix/overlays/gdal-small.nix @@ -0,0 +1,14 @@ +final: prev: { + # override the version of gdal used with postgis with the small version. + # significantly reduces overall closure size + gdal = prev.gdalMinimal.override { + /* other features can be enabled, reference: + https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/libraries/gdal/default.nix + */ + + # useHDF = true; + # useArrow = true; + # useLibHEIF = true; + # ... + }; +} diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix new file mode 100644 index 000000000..d55af10a2 --- /dev/null +++ b/nix/overlays/psql_16-oriole.nix @@ -0,0 +1,21 @@ +final: prev: { + postgresql_16 = prev.postgresql_16.overrideAttrs (old: { + pname = "postgresql_16"; + version = "16_23"; + src = prev.fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz"; + sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4="; + }; + buildInputs = old.buildInputs ++ [ + prev.bison + prev.docbook5 + prev.docbook_xsl + prev.docbook_xsl_ns + prev.docbook_xml_dtd_45 + prev.flex + prev.libxslt + prev.perl + ]; + }); + postgresql_orioledb_16 = final.postgresql_16; +} diff --git a/nix/tests/migrations/data.sql b/nix/tests/migrations/data.sql new file mode 100644 index 000000000..36396e6ad --- /dev/null +++ b/nix/tests/migrations/data.sql @@ -0,0 +1,21 @@ +create table account( + id int primary key, + is_verified bool, + name text, + phone text +); + +insert into public.account(id, is_verified, name, phone) +values + (1, true, 'foo', '1111111111'), + (2, true, 'bar', null), + (3, false, 'baz', '33333333333'); + +select id as test_new_key_id from pgsodium.create_key(name:='test_new_key') \gset + +select vault.create_secret ( + 's3kr3t_k3y', 'a_name', 'this is the foo secret key') test_secret_id \gset + +select vault.create_secret ( + 's3kr3t_k3y_2', 'another_name', 'this is another foo key', + (select id from pgsodium.key where name = 'test_new_key')) test_secret_id_2 \gset diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in new file mode 100644 index 000000000..91ace7d83 --- /dev/null +++ b/nix/tests/postgresql.conf.in @@ -0,0 +1,797 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory + # (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file + # (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' # what IP address(es) to listen on; +#port = @PGSQL_DEFAULT_PORT@ # (change requires restart) +max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +unix_socket_directories = '/tmp' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +#client_connection_check_interval = 0 # time between checks for client + # disconnection while running queries; + # 0 for never + +# - Authentication - + +#authentication_timeout = 1min # 1s-600s +#password_encryption = scram-sha-256 # scram-sha-256 or md5 +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +#ssl = off +#ssl_ca_file = '' +#ssl_cert_file = 'server.crt' +#ssl_crl_file = '' +#ssl_crl_dir = '' +#ssl_key_file = 'server.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_min_protocol_version = 'TLSv1.2' +#ssl_max_protocol_version = '' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#huge_page_size = 0 # zero for system default + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#logical_decoding_work_mem = 64MB # min 64kB +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) +#min_dynamic_shared_memory = 0MB # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 2 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 512kB # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#backend_flush_after = 0 # measured in pages, 0 disables +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#parallel_leader_participation = on +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +wal_level = logical # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +wal_log_hints = on # also do full page writes of non-critical updates + # (change requires restart) +#wal_compression = off # enable compression of full-page writes +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +#checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables +max_wal_size = 1GB +min_wal_size = 80MB + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the primary and on any standby that will send replication data. + +#max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +#max_replication_slots = 10 # max number of replication slots + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +#max_slot_wal_keep_size = -1 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Primary Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a primary server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from primary + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_async_append = on +#enable_bitmapscan = on +#enable_gathermerge = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_incremental_sort = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_memoize = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_parallel_hash = on +#enable_partition_pruning = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +#effective_cache_size = 4GB + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#jit = on # allow JIT compilation +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (Windows): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_autovacuum_min_duration = -1 # log autovacuum activity; + # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '%m [%p] ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %P = process ID of parallel group leader + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'America/Chicago' + + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +#cluster_name = '' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_activity_query_size = 1024 # (change requires restart) +#track_counts = on +#track_io_timing = off +#track_wal_io_timing = off +#track_functions = none # none, pl, all +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#compute_query_id = auto +#log_statement_stats = off +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +#row_security = on +#default_table_access_method = 'heap' +#default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#idle_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_table_age = 150000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'America/Chicago' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 1 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'C' # locale for system error message + # strings +lc_monetary = 'C' # locale for monetary formatting +lc_numeric = 'C' # locale for number formatting +lc_time = 'C' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#local_preload_libraries = '' +#session_preload_libraries = '' +#FIXME if we re-activate citus we'll need to add it back in here +#shared_preload_libraries = 'citus,auto_explain,pgsodium' +shared_preload_libraries = 'auto_explain,pgsodium' +#jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here + +pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql new file mode 100644 index 000000000..ba3e79823 --- /dev/null +++ b/nix/tests/prime.sql @@ -0,0 +1,23 @@ +CREATE EXTENSION IF NOT EXISTS pgtap; + +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS pgrouting; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +CREATE EXTENSION IF NOT EXISTS pgsodium; +CREATE EXTENSION IF NOT EXISTS supabase_vault; + +CREATE EXTENSION IF NOT EXISTS pgjwt; +CREATE EXTENSION IF NOT EXISTS plpgsql_check; +CREATE EXTENSION IF NOT EXISTS plv8; +CREATE EXTENSION IF NOT EXISTS rum; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pg_repack; +CREATE EXTENSION IF NOT EXISTS pgroonga; + +CREATE EXTENSION IF NOT EXISTS wrappers; +CREATE EXTENSION IF NOT EXISTS http; +CREATE EXTENSION IF NOT EXISTS pg_graphql; +CREATE EXTENSION IF NOT EXISTS pg_jsonschema; + +-- deactivate citus +-- CREATE EXTENSION IF NOT EXISTS citus; diff --git a/nix/tests/smoke/0000-hello-world.sql b/nix/tests/smoke/0000-hello-world.sql new file mode 100644 index 000000000..d6f002dec --- /dev/null +++ b/nix/tests/smoke/0000-hello-world.sql @@ -0,0 +1,10 @@ +-- Start transaction and plan the tests. +BEGIN; +SELECT plan(1); + +-- Run the tests. +SELECT pass( 'My test passed, w00t!' ); + +-- Finish the tests and clean up. +SELECT * FROM finish(); +ROLLBACK; diff --git a/nix/tests/smoke/0001-pg_graphql.sql b/nix/tests/smoke/0001-pg_graphql.sql new file mode 100644 index 000000000..80e3cb208 --- /dev/null +++ b/nix/tests/smoke/0001-pg_graphql.sql @@ -0,0 +1,59 @@ +-- Start transaction and plan the tests. +begin; + select plan(1); + + create extension if not exists pg_graphql; + + create table account( + id int primary key, + is_verified bool, + name text, + phone text + ); + + insert into public.account(id, is_verified, name, phone) + values + (1, true, 'foo', '1111111111'), + (2, true, 'bar', null), + (3, false, 'baz', '33333333333'); + + select is( + graphql.resolve($$ + { + accountCollection { + edges { + node { + id + } + } + } + } + $$), + '{ + "data": { + "accountCollection": { + "edges": [ + { + "node": { + "id": 1 + } + }, + { + "node": { + "id": 2 + } + }, + { + "node": { + "id": 3 + } + } + ] + } + } + }'::jsonb + ); + + + select * from finish(); +rollback; diff --git a/nix/tests/smoke/0002-supautils.sql b/nix/tests/smoke/0002-supautils.sql new file mode 100644 index 000000000..7a2160639 --- /dev/null +++ b/nix/tests/smoke/0002-supautils.sql @@ -0,0 +1,17 @@ +BEGIN; +SELECT plan(2); + +-- the setting doesn't exist when supautils is not loaded +SELECT throws_ok($$ + select current_setting('supautils.privileged_extensions', false) +$$); + +LOAD 'supautils'; + +-- now it does +SELECT ok( + current_setting('supautils.privileged_extensions', false) = '' +); + +SELECT * FROM finish(); +ROLLBACK; diff --git a/nix/tests/smoke/0003-pgsodium-vault.sql b/nix/tests/smoke/0003-pgsodium-vault.sql new file mode 100644 index 000000000..1c9cedf91 --- /dev/null +++ b/nix/tests/smoke/0003-pgsodium-vault.sql @@ -0,0 +1,40 @@ +BEGIN; + +select plan(3); + +select id as test_new_key_id from pgsodium.create_key(name:='test_new_key') \gset + +select vault.create_secret ( + 's3kr3t_k3y', 'a_name', 'this is the foo secret key') test_secret_id \gset + +select vault.create_secret ( + 's3kr3t_k3y_2', 'another_name', 'this is another foo key', + (select id from pgsodium.key where name = 'test_new_key')) test_secret_id_2 \gset + +SELECT results_eq( + $$ + SELECT decrypted_secret = 's3kr3t_k3y', description = 'this is the foo secret key' + FROM vault.decrypted_secrets WHERE name = 'a_name'; + $$, + $$VALUES (true, true)$$, + 'can select from masking view with custom key'); + +SELECT results_eq( + $$ + SELECT decrypted_secret = 's3kr3t_k3y_2', description = 'this is another foo key' + FROM vault.decrypted_secrets WHERE key_id = (select id from pgsodium.key where name = 'test_new_key'); + $$, + $$VALUES (true, true)$$, + 'can select from masking view'); + +SELECT lives_ok( + format($test$ + select vault.update_secret( + %L::uuid, new_name:='a_new_name', + new_secret:='new_s3kr3t_k3y', new_description:='this is the bar key') + $test$, :'test_secret_id'), + 'can update name, secret and description' + ); + +SELECT * FROM finish(); +ROLLBACK; diff --git a/nix/tests/util/pgsodium_getkey.sh b/nix/tests/util/pgsodium_getkey.sh new file mode 100755 index 000000000..778918ca1 --- /dev/null +++ b/nix/tests/util/pgsodium_getkey.sh @@ -0,0 +1,4 @@ +# NOTE (aseipp): just use some random key for testing, no need to query +# /dev/urandom. also helps ferrit out other random flukes, perhaps? + +echo -n 8359dafbba5c05568799c1c24eb6c2fbff497654bc6aa5e9a791c666768875a1 diff --git a/nix/tools/README.md b/nix/tools/README.md new file mode 100644 index 000000000..2606a57ee --- /dev/null +++ b/nix/tools/README.md @@ -0,0 +1,2 @@ +This directory just contains tools, but you can't run them directly. For the +sake of robustness, you should use `nix run` on this repository to do so. diff --git a/nix/tools/migrate-tool.sh.in b/nix/tools/migrate-tool.sh.in new file mode 100644 index 000000000..94eef85a8 --- /dev/null +++ b/nix/tools/migrate-tool.sh.in @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +[ ! -z "$DEBUG" ] && set -x + +# first argument is the old version; a path 15 or 16 +if [[ $1 == /nix/store* ]]; then + if [ ! -L "$1/receipt.json" ] || [ ! -e "$1/receipt.json" ]; then + echo "ERROR: $1 does not look like a valid Postgres install" + exit 1 + fi + OLDVER="$1" +elif [ "$1" == "15" ]; then + PSQL15=@PSQL15_BINDIR@ + OLDVER="$PSQL15" +elif [ "$1" == "16" ]; then + PSQL16=@PSQL16_BINDIR@ + OLDVER="$PSQL16" +else + echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" + exit 1 +fi + +# second argument is the new version; 15 or 16 +if [[ $2 == /nix/store* ]]; then + if [ ! -L "$2/receipt.json" ] || [ ! -e "$2/receipt.json" ]; then + echo "ERROR: $1 does not look like a valid Postgres install" + exit 1 + fi + NEWVER="$2" +elif [ "$2" == "15" ]; then + PSQL15=@PSQL15_BINDIR@ + NEWVER="$PSQL15" +elif [ "$2" == "16" ]; then + PSQL16=@PSQL16_BINDIR@ + NEWVER="$PSQL16" + echo "NEWVER IS $NEWVER" +else + echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path" + exit 1 +fi + +# thid argument is the upgrade method: either pg_dumpall or pg_ugprade +if [ "$3" != "pg_dumpall" ] && [ "$3" != "pg_upgrade" ]; then + echo "Please provide a valid upgrade method (pg_dumpall or pg_upgrade)" + exit 1 +fi +UPGRADE_METHOD="$3" + +echo "Old server build: PSQL $1" +echo "New server build: PSQL $2" +echo "Upgrade method: $UPGRADE_METHOD" + +PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" +DATDIR=$(mktemp -d) +NEWDAT=$(mktemp -d) +mkdir -p "$DATDIR" "$NEWDAT" + +echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed" +echo "NOTE: you are free to re-use this data directory at will" +echo + +$OLDVER/bin/initdb -D "$DATDIR" --locale=C +$NEWVER/bin/initdb -D "$NEWDAT" --locale=C + +# NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script +PSQL_CONF_FILE=@PSQL_CONF_FILE@ +PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ +echo "NOTE: patching postgresql.conf files" +for x in "$DATDIR" "$NEWDAT"; do + sed \ + "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + $PSQL_CONF_FILE > "$x/postgresql.conf" +done + +echo "NOTE: Starting first server (v${1}) to load data into the system" +$OLDVER/bin/pg_ctl start -D "$DATDIR" + +PRIMING_SCRIPT=@PRIMING_SCRIPT@ +MIGRATION_DATA=@MIGRATION_DATA@ + +$OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT" +$OLDVER/bin/psql -h localhost -d postgres -Xf "$MIGRATION_DATA" + +if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then + echo "NOTE: Stopping old server (v${1}) to prepare for migration" + $OLDVER/bin/pg_ctl stop -D "$DATDIR" + + echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade" + + export PGDATAOLD="$DATDIR" + export PGDATANEW="$NEWDAT" + export PGBINOLD="$OLDVER/bin" + export PGBINNEW="$NEWVER/bin" + + if ! $NEWVER/bin/pg_upgrade --check; then + echo "ERROR: pg_upgrade check failed" + exit 1 + fi + + echo "NOTE: pg_upgrade check passed, proceeding with migration" + $NEWVER/bin/pg_upgrade + rm -f delete_old_cluster.sh # we don't need this + exit 0 +fi + +if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then + SQLDAT="$DATDIR/dump.sql" + echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)" + $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT" + + echo "NOTE: Stopping old server (v${1}) to prepare for migration" + $OLDVER/bin/pg_ctl stop -D "$DATDIR" + + echo "NOTE: Starting second server (v${2}) to load data into the system" + $NEWVER/bin/pg_ctl start -D "$NEWDAT" + + echo "NOTE: Loading data into new server (v${2}) via 'cat | psql'" + cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres + + printf "\n\n\n\n" + echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT" + $NEWVER/bin/pg_ctl stop -D "$NEWDAT" +fi diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in new file mode 100644 index 000000000..e16198e0d --- /dev/null +++ b/nix/tools/run-client.sh.in @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +# first argument should be '15' or '16' for the version +if [ "$1" == "15" ]; then + echo "Starting client for PSQL 15" + PSQL15=@PSQL15_BINDIR@ + BINDIR="$PSQL15" +elif [ "$1" == "16" ]; then + echo "Starting client for PSQL 16" + PSQL16=@PSQL16_BINDIR@ + BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting client for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" +else + echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" + exit 1 +fi + +export PATH=$BINDIR/bin:$PATH + +PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" +PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ + +exec psql -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost postgres diff --git a/nix/tools/run-replica.sh.in b/nix/tools/run-replica.sh.in new file mode 100644 index 000000000..e2096b17a --- /dev/null +++ b/nix/tools/run-replica.sh.in @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +# first argument should be '15' or '16' for the version +if [ "$1" == "15" ]; then + echo "Starting server for PSQL 15" + PSQL15=@PSQL15_BINDIR@ + BINDIR="$PSQL15" +elif [ "$1" == "16" ]; then + echo "Starting server for PSQL 16" + PSQL16=@PSQL16_BINDIR@ + BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting server for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" +else + echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" + exit 1 +fi + +export PATH=$BINDIR/bin:$PATH + +PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ +MASTER_PORTNO="$2" +REPLICA_PORTNO="$3" +REPLICA_SLOT="replica_$RANDOM" +DATDIR=$(mktemp -d) +mkdir -p "$DATDIR" + +echo "NOTE: runing pg_basebackup for server on port $MASTER_PORTNO" +echo "NOTE: using replica slot $REPLICA_SLOT" + +pg_basebackup -p "$MASTER_PORTNO" -h localhost -U "${PGSQL_SUPERUSER}" -X stream -C -S "$REPLICA_SLOT" -v -R -D "$DATDIR" + +echo "NOTE: using port $REPLICA_PORTNO for replica" +echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" +echo "NOTE: you are free to re-use this data directory at will" +echo + +exec postgres -p "$REPLICA_PORTNO" -D "$DATDIR" -k /tmp diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in new file mode 100644 index 000000000..c5b6eb040 --- /dev/null +++ b/nix/tools/run-server.sh.in @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +# first argument should be '15' or '16' for the version +if [ "$1" == "15" ]; then + echo "Starting server for PSQL 15" + PSQL15=@PSQL15_BINDIR@ + BINDIR="$PSQL15" +elif [ "$1" == "16" ]; then + echo "Starting server for PSQL 16" + PSQL16=@PSQL16_BINDIR@ + BINDIR="$PSQL16" +elif [ "$1" == "orioledb-16" ]; then + echo "Starting server for PSQL ORIOLEDB 16" + PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ + BINDIR="$PSQLORIOLEDB16" +else + echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" + exit 1 +fi + +export PATH=$BINDIR/bin:$PATH + +PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ +PSQL_CONF_FILE=@PSQL_CONF_FILE@ +PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ +PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" +DATDIR=$(mktemp -d) +mkdir -p "$DATDIR" + +echo "NOTE: using port $PORTNO for server" +echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" +echo "NOTE: you are free to re-use this data directory at will" +echo + +initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C + +echo "NOTE: patching postgresql.conf files" +sed \ + "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" + +exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 24b048afdc48289265b9aafef66882dc9a4e7758 Mon Sep 17 00:00:00 2001 From: samrose Date: Sat, 16 Mar 2024 10:50:45 +0100 Subject: [PATCH 02/15] docs: updating docs to correspond with new location --- nix/docs/adding-tests.md | 7 ++-- nix/docs/build-postgres.md | 72 ++++----------------------------- nix/docs/new-major-postgres.md | 2 +- nix/docs/nix-overlays.md | 2 + nix/docs/start-client-server.md | 14 +++---- nix/docs/update-extension.md | 6 +-- nix/docs/use-direnv.md | 48 ---------------------- 7 files changed, 25 insertions(+), 126 deletions(-) diff --git a/nix/docs/adding-tests.md b/nix/docs/adding-tests.md index f417873a4..5f6612351 100644 --- a/nix/docs/adding-tests.md +++ b/nix/docs/adding-tests.md @@ -29,12 +29,13 @@ For a good example of a pgTAP test as a pull request, check out `nix flake check` gets its results cached, so if you do it again the tests won't rerun. If you change a file then it will run again. -If you want to force rerun without modifying a file, you can do: + Limitation: currently there's no way to rerun all the tests, so you have to specify the check attribute. @@ -57,7 +58,7 @@ extension authors), step 3 isn't guaranteed, so that's what the whole idea is designed to test. To add data into the database, modify the -[data.sql](../tests/migrations/data.sql) script and add whatever you want into +[data.sql](../nix/tests/migrations/data.sql) script and add whatever you want into it. This script gets loaded into the old version of the database at startup, and it's expected that the new version of the database can handle it. diff --git a/nix/docs/build-postgres.md b/nix/docs/build-postgres.md index 791bb98f6..2805d443a 100644 --- a/nix/docs/build-postgres.md +++ b/nix/docs/build-postgres.md @@ -1,10 +1,10 @@ -# 01 — Use this repository +# 01 — Using supabase nix Let's clone this repo: ```bash -git clone https://github.com/supabase/nix-postgres $HOME/tmp-nix-postgres -cd $HOME/tmp-nix-postgres +git clone https://github.com/supabase/postgres $HOME/supabase-postgres +cd $HOME/supabase-postgres ``` ## Hashes for everyone @@ -97,12 +97,12 @@ _new_ hash. The ability to refer to a piece of data by its hash, by some notion of _content_, is a very powerful primitive, as we'll see later. -## Build a different version: v14 +## Build a different version: v16 -What if we wanted PostgreSQL 14 and plugins? Just replace `_15` with `_14`: +What if we wanted PostgreSQL 16 and plugins? Just replace `_15` with `_16`: ``` -nix build .#psql_14/bin +nix build .#psql_16/bin ``` You're done: @@ -112,69 +112,13 @@ $ readlink result /nix/store/p7ziflx0000s28bfb213jsghrczknkc4-postgresql-and-plugins-14.8 ``` -## Do it all at once: using `just` - -But remembering that long name is tedious. (Don't worry, there's a method to -query the names, and we will go over it later.) What if we just wanted to build -something quickly? - -There's a great tool for this. It's called -[**Just**](https://github.com/casey/just), and all it does is make it easy to -run commands. Let's use that. - -Luckily, there's a `justfile` that can be used to build multiple things. Here's -the `build-all` rule in our [`justfile`](../justfile). It should be obvious if -you're familiar with `make` and `Makefile`s: - -``` -build-all: - nix build .#psql_14/bin .#psql_14/docker - nix build .#psql_15/bin .#psql_15/docker -``` - -So this actually builds _four_ things: v14 and v15 of Postgres, both as a binary -distribution _and_ as a Docker image. That's handy, instead of writing it all -out. So we run `just build-all`, and... - -``` -$ just build-all -Command 'just' not found, did you mean: -... -``` - -Ouch. So we have to install `just` using Cargo or something else first. Or do -we? ## Using `nix develop` -Nope! Here's how we can add `just` to our `$PATH`, transparently in a subshell: - -``` -$ which just - -$ echo $$ -766420 - -$ nix develop - -$ which just -/nix/store/1di6nb4qsv86907l3xarw4llzdss2g89-just-1.14.0/bin/just - -$ echo $$ -996868 - -$ just build-all -... - -$ exit - -$ echo $$ -766420 -``` -Done! As you can see, `nix develop .` will just drop you in a subshell with +`nix develop .` will just drop you in a subshell with tools you need _ready to go instantly_. That's all you need to do! And once that -shell goes away, `just` will be removed from your `$PATH` as well. +shell goes away, nix installed tools will be removed from your `$PATH` as well. There's an even easier way to do this [that is completely transparent to you, as well](./use-direnv.md). diff --git a/nix/docs/new-major-postgres.md b/nix/docs/new-major-postgres.md index d9d929273..1c5a2dffc 100644 --- a/nix/docs/new-major-postgres.md +++ b/nix/docs/new-major-postgres.md @@ -17,7 +17,7 @@ https://github.com/NixOS/nixpkgs/pull/249030 ## Adding the major version to this repository It isn't well abstracted, unfortunately. In short: look for the strings `14` and -`15` under `flake.nix` and `tools/`. More specifically: +`15` under `flake.nix` and `nix/tools/`. More specifically: - Add `psql_XX` to `basePackages` in `flake.nix` - Ditto with `checks` in `flake.nix` diff --git a/nix/docs/nix-overlays.md b/nix/docs/nix-overlays.md index eced66b98..90b6f221f 100644 --- a/nix/docs/nix-overlays.md +++ b/nix/docs/nix-overlays.md @@ -32,3 +32,5 @@ anything that depends on it. The names `final` and `prev` are used to refer to packages in terms of other overlays. For more information about this, see the [NixOS Wiki Page for Overlays](https://nixos.wiki/wiki/Overlays). + +We also use an overlay to override the default build recipe for `postgresql_16`, and instead feed it the specially patched postgres for use with orioledb extension. This experimental variant can be built with `nix build .#psql_orioledb_16/bin`. This will build this patched version of postgres, along with all extensions and wrappers that currently are known to work with orioledb. diff --git a/nix/docs/start-client-server.md b/nix/docs/start-client-server.md index 2bde53327..5f1d0d214 100644 --- a/nix/docs/start-client-server.md +++ b/nix/docs/start-client-server.md @@ -4,10 +4,10 @@ If you want to run a postgres server, just do this from the root of the repository: ``` -nix run .#start-server 14 +nix run .#start-server 15 ``` -Replace the `14` with a `15`, and you'll be using a different version. Optionally you can specify a second argument for the port. +Replace the `15` with a `16`, and you'll be using a different version. Optionally you can specify a second argument for the port. You likely have a running postgres, so to not cause a conflict, this uses port 5435 by default. @@ -17,7 +17,7 @@ valid "flake reference": ``` # from any arbitrary directory -nix run github:supabase/nix-postgres#start-server 14 +nix run github:supabase/postgres#start-server 15 ``` ### Arbitrary versions at arbitrary git revisions @@ -27,8 +27,8 @@ repository. You can change the syntax of the above to use _any_ version of the repository, at any time, by adding the commit hash after the repository name: ``` -# use postgresql 15 build at commit a9989d4800dd6038827afed27456f19ba4b2ae0a -nix run github:supabase/nix-postgres/a9989d4800dd6038827afed27456f19ba4b2ae0a#start-server 15 +# use postgresql 15 build at commit +nix run github:supabase/postgres/#start-server 15 ``` ## Running the client @@ -37,9 +37,9 @@ All of the same rules apply, but try using `start-client` on the right-hand side of the hash character, instead. For example: ``` -nix run github:supabase/nix-postgres#start-server 14 & +nix run github:supabase/postgres#start-server 15 & sleep 5 -nix run github:supabase/nix-postgres#start-client 15 +nix run github:supabase/postgres#start-client 16 ``` ## Running a server replica diff --git a/nix/docs/update-extension.md b/nix/docs/update-extension.md index f318a6e36..63efc895f 100644 --- a/nix/docs/update-extension.md +++ b/nix/docs/update-extension.md @@ -18,7 +18,7 @@ nix-update --flake psql_15/exts/pg_foobar git commit -asm "pg_foobar: update to latest release" ``` -It doesn't matter if you use `psql_14` or `psql_15` here, because `nix-update` +It doesn't matter if you use `psql_15` or `psql_16` here, because `nix-update` will look at the _file_ that extension is defined in, in order to update the source code. @@ -56,13 +56,13 @@ No extra arguments are necessary. You can specify an exact release tag using the `--version=` flag. Using the syntax `--version=branch` means "update to the latest version on the default branch." -## Example PRs + ## Other notes See issue [#5](https://github.com/supabase/nix-postgres/issues/5) for more diff --git a/nix/docs/use-direnv.md b/nix/docs/use-direnv.md index bad00ea64..cf34a2380 100644 --- a/nix/docs/use-direnv.md +++ b/nix/docs/use-direnv.md @@ -88,54 +88,6 @@ What just happened is that we populated the ambient shell environment with tools specified inside of `flake.nix` — we'll cover Flakes later. But for now, your tools are provisioned! -## Run `just` to build the code - -Previously, we needed to run `nix develop` to get access to the `just` command. -No longer! It's now available in your shell: - -``` -$ which just -/nix/store/1di6nb4qsv86907l3xarw4llzdss2g89-just-1.14.0/bin/just -``` - -``` -$ just b -... -``` - -Viola, done! - -## Go back `$HOME` and return again - -When you `cd ~`, direnv will automatically unload the `.envrc` file for you: - -``` -$ cd ~ -direnv: unloading -``` - -Now, `just` is no longer available: - -``` -$ just build-all -just: command not found -``` - -And if you go back, it is! - -``` -$ cd $HOME/nix-postgres -direnv: loading ~/work/nix-postgres/.envrc -direnv: loading ~/work/nix-postgres/.envrc.recommended -direnv: loading https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc (sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8=) -direnv: using flake -... -``` - -``` -$ just build-all -... -``` ## The power of `direnv` From 73bee62e6ec161055ba75056e9c9a4585f2cadc9 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 19 Mar 2024 10:39:35 +0700 Subject: [PATCH 03/15] feat: clearing out commented out citus references in the nix implementation --- flake.nix | 3 --- nix/ext/citus.nix | 37 ------------------------------------ nix/tests/postgresql.conf.in | 2 -- nix/tests/prime.sql | 3 --- 4 files changed, 45 deletions(-) delete mode 100644 nix/ext/citus.nix diff --git a/flake.nix b/flake.nix index d4c29fc20..9fba7a35f 100644 --- a/flake.nix +++ b/flake.nix @@ -131,9 +131,6 @@ # rollout new versions of these critical things easier without having to # go through the upstream release engineering process. ourExtensions = [ - # see comment in ./nix/ext/citus.nix - # for why citus is deactivated - #./nix/ext/citus.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix ./nix/ext/pg_net.nix diff --git a/nix/ext/citus.nix b/nix/ext/citus.nix deleted file mode 100644 index 0bb61ec31..000000000 --- a/nix/ext/citus.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ lib, curl, lz4, zstd, krb5, icu, stdenv, fetchFromGitHub, postgresql }: -#FIXME we have decided deactivate this for now, as it is not used in production -# if we decide to use it later we may need to look at this documentation -# https://docs.citusdata.com/en/stable/admin_guide/upgrading_citus.html#upgrading-postgresql-version-from-15-to-16 -stdenv.mkDerivation rec { - pname = "citus"; - version = "12.1.2"; - - buildInputs = [ curl lz4 zstd krb5 icu.dev postgresql ]; - - src = fetchFromGitHub { - owner = "citusdata"; - repo = pname; - rev = "refs/tags/v${version}"; - hash = "sha256-0uYNMLAYigtGlDRvOEkQeC5i58QfXcdSVjTQwWVFX+8="; - }; - - installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} - - cp src/backend/columnar/citus_columnar.so $out/lib - cp src/backend/columnar/citus_columnar.control $out/share/postgresql/extension - cp src/backend/columnar/build/sql/*.sql $out/share/postgresql/extension - - cp src/backend/distributed/citus.so $out/lib - cp src/backend/distributed/citus.control $out/share/postgresql/extension - cp src/backend/distributed/build/sql/*.sql $out/share/postgresql/extension - ''; - - meta = with lib; { - description = "Distributed PostgreSQL as an extension"; - homepage = "https://github.com/citusdata/${pname}"; - maintainers = with maintainers; [ olirice ]; - platforms = postgresql.meta.platforms; - license = licenses.agpl3Plus; - }; -} diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 91ace7d83..4c5075aa1 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -717,8 +717,6 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -#FIXME if we re-activate citus we'll need to add it back in here -#shared_preload_libraries = 'citus,auto_explain,pgsodium' shared_preload_libraries = 'auto_explain,pgsodium' #jit_provider = 'llvmjit' # JIT library to use diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index ba3e79823..283903110 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -18,6 +18,3 @@ CREATE EXTENSION IF NOT EXISTS wrappers; CREATE EXTENSION IF NOT EXISTS http; CREATE EXTENSION IF NOT EXISTS pg_graphql; CREATE EXTENSION IF NOT EXISTS pg_jsonschema; - --- deactivate citus --- CREATE EXTENSION IF NOT EXISTS citus; From 242c881fbc61e3dce4bc3be819debd8f7f61af05 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 19 Mar 2024 10:40:41 +0700 Subject: [PATCH 04/15] chore: fmt --- flake.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 9fba7a35f..79380c860 100644 --- a/flake.nix +++ b/flake.nix @@ -47,7 +47,7 @@ (import ./nix/overlays/cargo-pgrx.nix) (import ./nix/overlays/gdal-small.nix) (import ./nix/overlays/psql_16-oriole.nix) - + ]; }; #This variable works the same as 'oriole_pkgs' but builds using the upstream @@ -61,7 +61,7 @@ # explicit is better. (import ./nix/overlays/cargo-pgrx.nix) (import ./nix/overlays/gdal-small.nix) - + ]; }; @@ -79,7 +79,7 @@ --set LOCALE_ARCHIVE "${pkgs.glibcLocales}/lib/locale/locale-archive" done ''; - + # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by @@ -150,7 +150,7 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option - orioledbExtension = ourExtensions ++ [./nix/ext/orioledb.nix ]; + orioledbExtension = ourExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; From 651e1a2c3f6f018f839c3603217b59ffc81598be Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 19 Mar 2024 10:48:39 +0700 Subject: [PATCH 05/15] chore: update maintainer gh username --- nix/ext/hypopg.nix | 2 +- nix/ext/pg_graphql.nix | 2 +- nix/ext/pg_hashids.nix | 2 +- nix/ext/pg_jsonschema.nix | 2 +- nix/ext/pg_net.nix | 2 +- nix/ext/pg_plan_filter.nix | 2 +- nix/ext/pg_stat_monitor.nix | 2 +- nix/ext/pg_tle.nix | 2 +- nix/ext/pgsodium.nix | 2 +- nix/ext/pgsql-http.nix | 2 +- nix/ext/plv8.nix | 2 +- nix/ext/vault.nix | 2 +- nix/ext/wrappers/default.nix | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index ea41c57ee..79d8e189f 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Hypothetical Indexes for PostgreSQL"; homepage = "https://github.com/HypoPG/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_graphql.nix b/nix/ext/pg_graphql.nix index 7ef16e060..11cd67a97 100644 --- a/nix/ext/pg_graphql.nix +++ b/nix/ext/pg_graphql.nix @@ -22,7 +22,7 @@ buildPgrxExtension rec { meta = with lib; { description = "GraphQL support for PostreSQL"; homepage = "https://github.com/supabase/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 9d237d09c..af3f1740c 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Generate short unique IDs in PostgreSQL"; homepage = "https://github.com/iCyberon/pg_hashids"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix index df357f337..6d4b45db0 100644 --- a/nix/ext/pg_jsonschema.nix +++ b/nix/ext/pg_jsonschema.nix @@ -21,7 +21,7 @@ buildPgrxExtension rec { meta = with lib; { description = "JSON Schema Validation for PostgreSQL"; homepage = "https://github.com/supabase/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index 919ef8b4a..a4497e68b 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Async networking for Postgres"; homepage = "https://github.com/supabase/pg_net"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 49d839fcc..12890d3b6 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Filter PostgreSQL statements by execution plans"; homepage = "https://github.com/pgexperts/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 35e6b3bfa..954590607 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -43,7 +43,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Query Performance Monitoring Tool for PostgreSQL"; homepage = "https://github.com/percona/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; broken = lib.versionOlder postgresql.version "15"; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 626cd8a89..6188c7ce1 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Framework for 'Trusted Language Extensions' in PostgreSQL"; homepage = "https://github.com/aws/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index d97aa80c7..e3b0da228 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Modern cryptography for PostgreSQL"; homepage = "https://github.com/michelp/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index e94e2ba9d..2edaa9d53 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -24,7 +24,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "HTTP client for Postgres"; homepage = "https://github.com/pramsey/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 0ff1fb4dc..bc92dc7ab 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -139,7 +139,7 @@ stdenv.mkDerivation (finalAttrs: { meta = with lib; { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; - maintainers = with maintainers; [ marsam ]; + maintainers = with maintainers; [ samrose ]; platforms = [ "x86_64-linux" "aarch64-linux" ]; license = licenses.postgresql; broken = postgresql.jitSupport; diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index 4e2684ab2..c822fcd51 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Store encrypted secrets in PostgreSQL"; homepage = "https://github.com/supabase/${pname}"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 89299a2bd..dcab7de3a 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -54,7 +54,7 @@ buildPgrxExtension_0_11_2 rec { meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; homepage = "https://github.com/supabase/wrappers"; - maintainers = with maintainers; [ thoughtpolice ]; + maintainers = with maintainers; [ samrose ]; platforms = postgresql.meta.platforms; license = licenses.postgresql; }; From 28fa350acf5f791e7efedf7c45ed31c428872807 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 19 Mar 2024 10:58:58 +0700 Subject: [PATCH 06/15] feat: update pgvector to 0.6.0 --- nix/ext/pgvector.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 996a671ff..1361aa3f1 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgvector"; - version = "0.5.1"; + version = "0.6.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "pgvector"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-ZNzq+dATZn9LUgeOczsaadr5hwdbt9y/+sAOPIdr77U="; + hash = "sha256-hXm+k0BZ9xZP1Tnek14jPoKCPQkA5ovscu9IX2mW7Kc="; }; installPhase = '' From 28be8d06c6753b90f308439f6343b8938d9fed28 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 20 Mar 2024 14:37:36 +0700 Subject: [PATCH 07/15] feat: include nix specific ci (may require env vars configured in repo settings) --- .github/workflows/nix-build.yml | 26 +++++++++++++++ .github/workflows/nix-cache-upload.yml | 46 ++++++++++++++++++++++++++ .github/workflows/nix-docker-image.yml | 43 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 .github/workflows/nix-build.yml create mode 100644 .github/workflows/nix-cache-upload.yml create mode 100644 .github/workflows/nix-docker-image.yml diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml new file mode 100644 index 000000000..11b88baf8 --- /dev/null +++ b/.github/workflows/nix-build.yml @@ -0,0 +1,26 @@ +name: Nix CI + +on: + push: + branches: + - main + pull_request: + +permissions: read-all + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + cmd: [ "nix flake check -L --show-trace", "nix run nixpkgs#just -- build-all" ] + runs-on: ${{ matrix.os }} + name: nix-build + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + with: + fetch-depth: 0 + - uses: DeterminateSystems/nix-installer-action@65d7c888b2778e8cf30a07a88422ccb23499bfb8 + - uses: DeterminateSystems/magic-nix-cache-action@749fc5bbc9fa49d60c2b93f6c4bc867b82e1d295 + - run: ${{ matrix.cmd }} diff --git a/.github/workflows/nix-cache-upload.yml b/.github/workflows/nix-cache-upload.yml new file mode 100644 index 000000000..cd162f2cd --- /dev/null +++ b/.github/workflows/nix-cache-upload.yml @@ -0,0 +1,46 @@ +name: Nix Cache upload + +on: + push: + branches: + - main + +permissions: + contents: write + packages: write + id-token: write + +jobs: + build: + strategy: + fail-fast: false + runs-on: ubuntu-latest + name: nix-build + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + with: + fetch-depth: 0 + - uses: DeterminateSystems/nix-installer-action@65d7c888b2778e8cf30a07a88422ccb23499bfb8 + - uses: DeterminateSystems/magic-nix-cache-action@749fc5bbc9fa49d60c2b93f6c4bc867b82e1d295 + - name: configure aws credentials for s3 + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + kvm: true + extra-conf: | + system-features = kvm + + - name: write secret key + # use python so we don't interpolate the secret into the workflow logs, in case of bugs + run: | + python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" + env: + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + + - name: build and copy to S3 + run: | + for x in 15 16 orioledb_16; do + nix build .#psql_$x/bin -o result-$x + done + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./result* diff --git a/.github/workflows/nix-docker-image.yml b/.github/workflows/nix-docker-image.yml new file mode 100644 index 000000000..8a83c1a5c --- /dev/null +++ b/.github/workflows/nix-docker-image.yml @@ -0,0 +1,43 @@ +name: Nix Docker + +on: + push: + branches: [ 'main' ] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + name: "update: build and deploy postgres server images" + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + with: + fetch-depth: 0 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: actions/checkout@v3 + + - name: Build images + run: | + nix build .#psql_15/docker -o result-docker-15 + nix build .#psql_16/docker -o result-docker-16 + nix build .#psql_orioledb_16/docker -o result-docker-orioledb-16 + - name: Log in to the Container registry + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push image to the Container registry + run: | + set -x + for x in 15 16 orioledb_16; do + nix build .#psql_$x/docker.copyToRegistry + done From 0ba5fdf7d0bc72127dadf617d94ed085c15a6f7b Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 20 Mar 2024 15:46:47 +0700 Subject: [PATCH 08/15] fix: deprecate justfile --- .github/workflows/nix-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 11b88baf8..72ce14e3d 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest ] - cmd: [ "nix flake check -L --show-trace", "nix run nixpkgs#just -- build-all" ] + cmd: [ "nix flake check -L --show-trace", "nix build .#psql_15/bin .#psql_15/docker", "nix build .#psql_16/bin .#psql_16/docker", "nix build .#psql_orioledb_16/bin .#psql_orioledb_16/docker"] runs-on: ${{ matrix.os }} name: nix-build steps: From 8e8c8a712056f632ac1b3d24a86023440509582a Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 27 Mar 2024 19:55:55 -0400 Subject: [PATCH 09/15] feat: wip update versions from ansible vars data --- flake.lock | 57 +++++++++++++++++++++++++++-- flake.nix | 15 +++++++- nix/tools/sync-exts-versions.sh.in | 58 ++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 nix/tools/sync-exts-versions.sh.in diff --git a/flake.lock b/flake.lock index 59e9002bb..3de027643 100644 --- a/flake.lock +++ b/flake.lock @@ -36,10 +36,29 @@ "type": "github" } }, + "nix-editor": { + "inputs": { + "nixpkgs": "nixpkgs", + "utils": "utils" + }, + "locked": { + "lastModified": 1703105021, + "narHash": "sha256-Ne9NG7x45a8aJyAN+yYWbr/6mQHBVVkwZZ72EZHHRqw=", + "owner": "snowfallorg", + "repo": "nix-editor", + "rev": "b5017f8d61753ce6a3a1a2aa7e474d59146a8ae3", + "type": "github" + }, + "original": { + "owner": "snowfallorg", + "repo": "nix-editor", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" }, "locked": { "lastModified": 1708764364, @@ -56,6 +75,22 @@ } }, "nixpkgs": { + "locked": { + "lastModified": 1675673983, + "narHash": "sha256-8hzNh1jtiPxL5r3ICNzSmpSzV7kGb3KwX+FS5BWJUTo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "5a350a8f31bb7ef0c6e79aea3795a890cf7743d4", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1697269602, "narHash": "sha256-dSzV7Ud+JH4DPVD9od53EgDrxUVQOcSj4KGjggCDVJI=", @@ -70,7 +105,7 @@ "type": "github" } }, - "nixpkgs_2": { + "nixpkgs_3": { "locked": { "lastModified": 1709386671, "narHash": "sha256-VPqfBnIJ+cfa78pd4Y5Cr6sOWVW8GYHRVucxJGmRf8Q=", @@ -89,8 +124,9 @@ "root": { "inputs": { "flake-utils": "flake-utils", + "nix-editor": "nix-editor", "nix2container": "nix2container", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_3" } }, "systems": { @@ -122,6 +158,21 @@ "repo": "default", "type": "github" } + }, + "utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 79380c860..25727f4c9 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; flake-utils.url = "github:numtide/flake-utils"; nix2container.url = "github:nlewo/nix2container"; + nix-editor.url = "github:snowfallorg/nix-editor"; }; nixConfig = { @@ -17,7 +18,7 @@ ]; }; - outputs = { self, nixpkgs, flake-utils, nix2container }: + outputs = { self, nixpkgs, flake-utils, nix2container, nix-editor, ...}: let gitRev = "vcs=${self.shortRev or "dirty"}+${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}"; @@ -477,6 +478,16 @@ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' chmod +x $out/bin/start-postgres-replica ''; + sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' + mkdir -p $out/bin + substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ + --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ + --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ + --subst-var-by 'NIX_EDITOR' '${nix-editor}/bin/nix-editor' \ + --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_14}/bin/nix-prefetch-url' \ + --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_14}/bin/nix' + chmod +x $out/bin/sync-exts-versions + ''; }; # Create a testing harness for a PostgreSQL package. This is used for @@ -545,6 +556,7 @@ start-client = mkApp "start-client" "start-postgres-client"; start-replica = mkApp "start-replica" "start-postgres-replica"; migration-test = mkApp "migrate-tool" "migrate-postgres"; + sync-ext-versions = mkApp "sync-exts-versions" "sync-ext-versions"; }; # 'devShells.default' lists the set of packages that are included in the @@ -563,6 +575,7 @@ basePackages.start-client basePackages.start-replica basePackages.migrate-tool + basePackages.sync-exts-versions ]; shellHook = '' export HISTFILE=.history diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in new file mode 100644 index 000000000..b9d0070ef --- /dev/null +++ b/nix/tools/sync-exts-versions.sh.in @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +#pass in env vars supplied by nix +yq=@YQ@ +jq=@JQ@ +editor=@NIX_EDITOR@ +ansible_vars=$($yq '.' $PWD/ansible/vars.yml) +prefetchurl=@NIXPREFETCHURL@ +_nix=@NIX@ +fetch_source_url() { + local source_url=${1//\"/} # Remove double quotes + source_url=${source_url//\'/} # Remove single quotes + + # Check if the source URL is provided + if [ -z "$source_url" ]; then + echo "Usage: fetch_nix_url " + return 1 + fi + + echo "$source_url" + + # Run nix-prefetch-url command + local val=$($prefetchurl --type sha256 "$source_url" | cut -d ' ' -f 2) + echo $val +} + +#iterate values in ansible vars, case statement +# to match ansible var to package name +keys=$(echo "$ansible_vars" | $jq -r 'keys[]') + +for key in $keys; do + case $key in + "pgvector_release") + value=$(echo $ansible_vars | $jq -r '.pgvector_release') + echo "$key: $value" + url=$($_nix eval .#psql_16/exts/pgvector.src.url) + hash=$(fetch_source_url $url | tail -n 1) + echo "$hash" + + ;; + *) + ;; + esac +done + +# url=$($_nix eval .#psql_16/exts/pgvector.src.url) + +# fetch_nix_url "$url" + +#res=$editor /home/sam/postgres/nix/ext/pgvector.nix src +#echo $res +# url=$($_nix eval .#psql_16/exts/pgvector.src.url) +# #echo $url +# hash=$(fetch_source_url $url | tail -n 1) +# echo "$hash" From bebc0f1890ebe4668f445209e7bf4b8ff2146bcc Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 27 Mar 2024 20:06:46 -0400 Subject: [PATCH 10/15] chore: stashing progress --- nix/tools/sync-exts-versions.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in index b9d0070ef..8828e5dcc 100644 --- a/nix/tools/sync-exts-versions.sh.in +++ b/nix/tools/sync-exts-versions.sh.in @@ -24,7 +24,7 @@ fetch_source_url() { # Run nix-prefetch-url command local val=$($prefetchurl --type sha256 "$source_url" | cut -d ' ' -f 2) - echo $val + echo "$val" } #iterate values in ansible vars, case statement From 4939b1f9df17d1eac807474a6344e0a541ecdb6e Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 29 Mar 2024 10:56:48 -0400 Subject: [PATCH 11/15] feat: WIP tooling that will update versions, source and cargovendor sha --- flake.nix | 2 +- nix/ext/pg_graphql.nix | 8 ++-- nix/tools/sync-exts-versions.sh.in | 69 ++++++++++++++++++++++++++---- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 25727f4c9..f57867098 100644 --- a/flake.nix +++ b/flake.nix @@ -483,7 +483,7 @@ substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ - --subst-var-by 'NIX_EDITOR' '${nix-editor}/bin/nix-editor' \ + --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \ --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_14}/bin/nix-prefetch-url' \ --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_14}/bin/nix' chmod +x $out/bin/sync-exts-versions diff --git a/nix/ext/pg_graphql.nix b/nix/ext/pg_graphql.nix index 11cd67a97..2cda9ae46 100644 --- a/nix/ext/pg_graphql.nix +++ b/nix/ext/pg_graphql.nix @@ -2,17 +2,17 @@ buildPgrxExtension rec { pname = "pg_graphql"; - version = "unstable-1.5.0"; + version = "1.5.1"; inherit postgresql; src = fetchFromGitHub { owner = "supabase"; repo = pname; - rev = "v1.5.0"; - hash = "sha256-28ANRZyF22qF2YAxNAAkPfGOM3+xiO6IHdXsTp0CTQE="; + rev = "v${version}"; + hash = "sha256-cAiD2iSFmZwC+Zy0x+MABseWCxXRtRY74Dj0oBKet+o="; }; - cargoHash = "sha256-CUiGs0m9aUeqjpdPyOSjz91cP7TT6kjJqnw7ImGnQuo="; + cargoHash = "sha256-BOw4SafWD8z2Oj8KPVv7GQEOQ1TCShhcG7XmQiM9W68="; # FIXME (aseipp): disable the tests since they try to install .control # files into the wrong spot, aside from that the one main test seems diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in index 8828e5dcc..53dae35ab 100644 --- a/nix/tools/sync-exts-versions.sh.in +++ b/nix/tools/sync-exts-versions.sh.in @@ -23,8 +23,53 @@ fetch_source_url() { echo "$source_url" # Run nix-prefetch-url command - local val=$($prefetchurl --type sha256 "$source_url" | cut -d ' ' -f 2) - echo "$val" + local initial_hash=$($prefetchurl --type sha256 "$source_url" --unpack | cut -d ' ' -f 2) + #once we can bump up nix version, we can use nix hash convert --hash-algo sha256 + local final_hash=$($_nix hash to-sri --type sha256 $initial_hash) + echo "$final_hash" +} + +sync_version() { + + local package_name=$1 + local version="\"$2\"" + local hash="\"$3\"" + + + # Update the version and hash in the Nix expression + $editor $PWD/nix/ext/$package_name.nix version --inplace -v "$version" + $editor $PWD/nix/ext/$package_name.nix src.hash --inplace -v $hash +} + +run_sync() { + local varname=$1 + local package_name=$2 + + version=$(echo $ansible_vars | $jq -r '.'$varname'') + echo "$key: $version" + url=$($_nix eval .#psql_16/exts/$package_name.src.url) + hash=$(fetch_source_url $url | tail -n 1) + $(sync_version $package_name $version $hash) + echo "synced $package_name to version $version with hash $hash" + + +} + +update_cargo_vendor_hash() { + local package_name=$1 + output=$($_nix build .#psql_16/exts/$package_name 2>&1) + + # Check if the command exited with an error + if [ $? -ne 0 ]; then + # Extract the hash value after "got: " + hash_value_scraped=$(echo "$output" | grep "got:" | awk '{for (i=1; i<=NF; i++) if ($i ~ /^sha/) print $i}') + hash_value="\"$hash_value_scraped\"" + # Continue using the captured hash value + $editor $PWD/nix/ext/$package_name.nix cargoHash --inplace -v $hash_value + echo "Updated cargoHash for $package_name to $hash_value" + else + echo "$package_name builds successfully, moving on..." + fi } #iterate values in ansible vars, case statement @@ -34,12 +79,20 @@ keys=$(echo "$ansible_vars" | $jq -r 'keys[]') for key in $keys; do case $key in "pgvector_release") - value=$(echo $ansible_vars | $jq -r '.pgvector_release') - echo "$key: $value" - url=$($_nix eval .#psql_16/exts/pgvector.src.url) - hash=$(fetch_source_url $url | tail -n 1) - echo "$hash" - + varname="pgvector_release" + package_name="pgvector" + run_sync $varname $package_name + ;; + "hypopg_release") + varname="hypopg_release" + package_name="hypopg" + run_sync $varname $package_name + ;; + "pg_graphql_release") + varname="pg_graphql_release" + package_name="pg_graphql" + run_sync $varname $package_name + update_cargo_vendor_hash $package_name ;; *) ;; From ff70c857578e3e1d6bae3d07591aa2f45140d3de Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 29 Mar 2024 17:29:45 -0400 Subject: [PATCH 12/15] feat: more work toward versions sync --- flake.nix | 17 ++++++----------- nix/ext/pg_hashids.nix | 4 ++-- nix/ext/pg_jsonschema.nix | 12 ++++++------ nix/ext/wrappers/default.nix | 15 ++++++++------- nix/overlays/cargo-pgrx-0-10-2.nix | 27 +++++++++++++++++++++++++++ nix/overlays/cargo-pgrx.nix | 12 ++++++------ nix/tools/sync-exts-versions.sh.in | 23 ++++++++++++++++++----- 7 files changed, 73 insertions(+), 37 deletions(-) create mode 100644 nix/overlays/cargo-pgrx-0-10-2.nix diff --git a/flake.nix b/flake.nix index f57867098..fd0e6de1a 100644 --- a/flake.nix +++ b/flake.nix @@ -61,6 +61,7 @@ # want to have an arbitrary order, since it might matter. being # explicit is better. (import ./nix/overlays/cargo-pgrx.nix) + (import ./nix/overlays/cargo-pgrx-0-10-2.nix) (import ./nix/overlays/gdal-small.nix) ]; @@ -414,8 +415,8 @@ basePackages = { # PostgreSQL versions. psql_15 = makePostgres "15"; - psql_16 = makePostgres "16"; - psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + #psql_16 = makePostgres "16"; + #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; # Start a version of the server. start-server = @@ -429,8 +430,6 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' @@ -444,8 +443,6 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb_16.bin}' chmod +x $out/bin/start-postgres-client ''; @@ -461,7 +458,6 @@ mkdir -p $out/bin substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ @@ -475,7 +471,6 @@ substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ - --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' chmod +x $out/bin/start-postgres-replica ''; sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' @@ -529,7 +524,7 @@ # set can go here. inherit (pkgs) # NOTE: comes from our cargo-pgrx.nix overlay - cargo-pgrx_0_11_2; + cargo-pgrx_0_11_0 cargo-pgrx_0_10_2; }; @@ -537,8 +532,8 @@ # flake check'. This is run in the CI system, as well. checks = { psql_15 = makeCheckHarness basePackages.psql_15.bin; - psql_16 = makeCheckHarness basePackages.psql_16.bin; - psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; + #psql_16 = makeCheckHarness basePackages.psql_16.bin; + #psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; }; # Apps is a list of names of things that can be executed with 'nix run'; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index af3f1740c..41c3ba664 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -2,14 +2,14 @@ stdenv.mkDerivation rec { pname = "pg_hashids"; - version = "unstable-2022-09-17"; + version = "cd0e1b31d52b394a0df64079406a14a4f7387cd6"; buildInputs = [ postgresql ]; src = fetchFromGitHub { owner = "iCyberon"; repo = pname; - rev = "cd0e1b31d52b394a0df64079406a14a4f7387cd6"; + rev = "${version}"; hash = "sha256-Nmb7XLqQflYZfqj0yrewfb1Hl5YgEB5wfjBunPwIuOU="; }; diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix index 6d4b45db0..afd2aa921 100644 --- a/nix/ext/pg_jsonschema.nix +++ b/nix/ext/pg_jsonschema.nix @@ -1,18 +1,18 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension }: +{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension_0_10_2 }: -buildPgrxExtension rec { +buildPgrxExtension_0_10_2 rec { pname = "pg_jsonschema"; - version = "unstable-v0.3.0"; + version = "0.2.0"; inherit postgresql; src = fetchFromGitHub { owner = "supabase"; repo = pname; - rev = "v0.3.0"; - hash = "sha256-am6Ye+pOoAsOr9L4vJXw4iIOJ9x0VkUjqH6PdXMUZrk="; + rev = "v${version}"; + hash = "sha256-57gZbUVi8P4EB8T0P19JBVXcetQcr6IxuIx96NNFA/0="; }; - cargoHash = "sha256-tiiWzu/mTKL5ruvWn6IxrXVhVqS4LXzjfacdFT9rbOY="; + cargoHash = "sha256-XMKZUdubCwPDnrv6yao7GaavoRK6cVyy1WqFBQ6wh3s="; # FIXME (aseipp): testsuite tries to write files into /nix/store; we'll have # to fix this a bit later. diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index dcab7de3a..534f9d7a5 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -4,20 +4,19 @@ , openssl , pkg-config , postgresql -, buildPgrxExtension_0_11_2 +, buildPgrxExtension_0_11_0 }: -buildPgrxExtension_0_11_2 rec { +buildPgrxExtension_0_11_0 rec { pname = "supabase-wrappers"; - version = "unstable-2024-02-26"; + version = "0.2.0"; inherit postgresql; src = fetchFromGitHub { owner = "supabase"; repo = "wrappers"; - #rev pinned for now to the HEAD of the main branch to achieve cargo-pgrx 0.11.2 compat - rev = "5b5c2622268c75bec834a38b2ff967f781511188"; - hash = "sha256-VwEFJD0yD+gvXCTzq9NfjCPEkh/lDQdEOPfk8LwK4z4="; + rev = "v${version}"; + hash = "sha256-F+S5uyubL3Tb3RTJ08Zf9gN8oLE/WkCWFA8RcKkDqes="; }; nativeBuildInputs = [ pkg-config ]; @@ -27,7 +26,9 @@ buildPgrxExtension_0_11_2 rec { OPENSSL_NO_VENDOR = 1; cargoLock = { - lockFile = "${src}/Cargo.lock"; + #TODO when we move to newer versions this lockfile will need to be sourced + # from ${src}/Cargo.lock + lockFile = "${src}/wrappers/Cargo.lock"; outputHashes = { "clickhouse-rs-1.0.0-alpha.1" = "sha256-0zmoUo/GLyCKDLkpBsnLAyGs1xz6cubJhn+eVqMEMaw="; }; diff --git a/nix/overlays/cargo-pgrx-0-10-2.nix b/nix/overlays/cargo-pgrx-0-10-2.nix new file mode 100644 index 000000000..b07dacaf2 --- /dev/null +++ b/nix/overlays/cargo-pgrx-0-10-2.nix @@ -0,0 +1,27 @@ +final: prev: { + cargo-pgrx_0_10_2 = prev.cargo-pgrx.overrideAttrs (oldAttrs: rec { + pname = "cargo-pgrx"; + version = "0.10.2"; + + src = prev.fetchCrate { + inherit version pname; + hash = "sha256-FqjfbJmSy5UCpPPPk4bkEyvQCnaH9zYtkI7txgIn+ls="; + }; + + # NOTE (aseipp): normally, we would just use 'cargoHash' here, but + # due to a fantastic interaction of APIs, we can't do that so + # easily, and have to use this incantation instead, which is + # basically the exact same thing but with 4 extra lines. see: + # + # https://discourse.nixos.org/t/is-it-possible-to-override-cargosha256-in-buildrustpackage/4393/5 + cargoDeps = oldAttrs.cargoDeps.overrideAttrs (prev.lib.const { + name = "${pname}-vendor.tar.gz"; + inherit src; + outputHash = "sha256-0blBUEm8PPbDyF+NnSwoMJpu+a20zq1/2+dzP0H9i+E="; + }); + }); + + buildPgrxExtension_0_10_2 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx_0_10_2; + }; +} diff --git a/nix/overlays/cargo-pgrx.nix b/nix/overlays/cargo-pgrx.nix index 7254d2908..70d63060f 100644 --- a/nix/overlays/cargo-pgrx.nix +++ b/nix/overlays/cargo-pgrx.nix @@ -1,11 +1,11 @@ final: prev: { - cargo-pgrx_0_11_2 = prev.cargo-pgrx.overrideAttrs (oldAttrs: rec { + cargo-pgrx_0_11_0 = prev.cargo-pgrx.overrideAttrs (oldAttrs: rec { pname = "cargo-pgrx"; - version = "0.11.2"; + version = "0.11.0"; src = prev.fetchCrate { inherit version pname; - hash = "sha256-8NlpMDFaltTIA8G4JioYm8LaPJ2RGKH5o6sd6lBHmmM="; + hash = "sha256-GiUjsSqnrUNgiT/d3b8uK9BV7cHFvaDoq6cUGRwPigM="; }; # NOTE (aseipp): normally, we would just use 'cargoHash' here, but @@ -17,11 +17,11 @@ final: prev: { cargoDeps = oldAttrs.cargoDeps.overrideAttrs (prev.lib.const { name = "${pname}-vendor.tar.gz"; inherit src; - outputHash = "sha256-qU2r67qI+aWsWr3vMWHb2FItHzwSaqXDnTvRe0rf+JY="; + outputHash = "sha256-DB+MQaTj5HWsIxrk5mJblBeGaI4qOuuV24AdjT3ES3o="; }); }); - buildPgrxExtension_0_11_2 = prev.buildPgrxExtension.override { - cargo-pgrx = final.cargo-pgrx_0_11_2; + buildPgrxExtension_0_11_0 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx_0_11_0; }; } diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in index 53dae35ab..322f2f40b 100644 --- a/nix/tools/sync-exts-versions.sh.in +++ b/nix/tools/sync-exts-versions.sh.in @@ -47,7 +47,7 @@ run_sync() { version=$(echo $ansible_vars | $jq -r '.'$varname'') echo "$key: $version" - url=$($_nix eval .#psql_16/exts/$package_name.src.url) + url=$($_nix eval .#psql_15/exts/$package_name.src.url) hash=$(fetch_source_url $url | tail -n 1) $(sync_version $package_name $version $hash) echo "synced $package_name to version $version with hash $hash" @@ -57,7 +57,8 @@ run_sync() { update_cargo_vendor_hash() { local package_name=$1 - output=$($_nix build .#psql_16/exts/$package_name 2>&1) + $editor $PWD/nix/ext/$package_name.nix cargoHash --inplace -v "" + output=$($_nix build .#psql_15/exts/$package_name 2>&1) # Check if the command exited with an error if [ $? -ne 0 ]; then @@ -78,11 +79,12 @@ keys=$(echo "$ansible_vars" | $jq -r 'keys[]') for key in $keys; do case $key in - "pgvector_release") - varname="pgvector_release" - package_name="pgvector" + "pg_hashids_release") + varname="pg_hashids_release" + package_name="pg_hashids" run_sync $varname $package_name ;; + "hypopg_release") varname="hypopg_release" package_name="hypopg" @@ -94,6 +96,17 @@ for key in $keys; do run_sync $varname $package_name update_cargo_vendor_hash $package_name ;; + "pg_jsonschema_release") + varname="pg_jsonschema_release" + package_name="pg_jsonschema" + run_sync $varname $package_name + update_cargo_vendor_hash $package_name + ;; + "pgvector_release") + varname="pgvector_release" + package_name="pgvector" + run_sync $varname $package_name + ;; *) ;; esac From 2fe314db022e4423dde9813e92c5fa56ad4fc225 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 29 Mar 2024 17:34:04 -0400 Subject: [PATCH 13/15] fix: gh action targets --- .github/workflows/nix-build.yml | 2 +- .github/workflows/nix-docker-image.yml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 72ce14e3d..a6d8ad116 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest ] - cmd: [ "nix flake check -L --show-trace", "nix build .#psql_15/bin .#psql_15/docker", "nix build .#psql_16/bin .#psql_16/docker", "nix build .#psql_orioledb_16/bin .#psql_orioledb_16/docker"] + cmd: [ "nix flake check -L --show-trace", "nix build .#psql_15/bin .#psql_15/docker"] runs-on: ${{ matrix.os }} name: nix-build steps: diff --git a/.github/workflows/nix-docker-image.yml b/.github/workflows/nix-docker-image.yml index 8a83c1a5c..132414b37 100644 --- a/.github/workflows/nix-docker-image.yml +++ b/.github/workflows/nix-docker-image.yml @@ -26,8 +26,6 @@ jobs: - name: Build images run: | nix build .#psql_15/docker -o result-docker-15 - nix build .#psql_16/docker -o result-docker-16 - nix build .#psql_orioledb_16/docker -o result-docker-orioledb-16 - name: Log in to the Container registry uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 with: @@ -38,6 +36,6 @@ jobs: - name: Push image to the Container registry run: | set -x - for x in 15 16 orioledb_16; do + for x in 15; do nix build .#psql_$x/docker.copyToRegistry done From 449849cf7a741c161c459fe588ecf714d2a26a8d Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 1 Apr 2024 11:16:38 -0400 Subject: [PATCH 14/15] fix: conform name --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index fd0e6de1a..0e6c53d0d 100644 --- a/flake.nix +++ b/flake.nix @@ -551,7 +551,7 @@ start-client = mkApp "start-client" "start-postgres-client"; start-replica = mkApp "start-replica" "start-postgres-replica"; migration-test = mkApp "migrate-tool" "migrate-postgres"; - sync-ext-versions = mkApp "sync-exts-versions" "sync-ext-versions"; + sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; }; # 'devShells.default' lists the set of packages that are included in the From f0a107f1eadfb498b2b9e178943fb8479ae46450 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 1 Apr 2024 15:21:03 -0400 Subject: [PATCH 15/15] feat: bring all exts into local/native build (no more upstream) run the sync util on the nix expressions, and then build and test against those versions as synced from ansible/vars.yml paring down to psql 15.6 + extensions/wrappers (put aside oriole and psql 16 for now, will bring back when ready) --- README.md | 9 ++ flake.nix | 41 +++----- nix/ext/pg-safeupdate.nix | 29 ++++++ nix/ext/pg_cron.nix | 32 ++++++ nix/ext/pg_net.nix | 6 +- nix/ext/pg_plan_filter.nix | 4 +- nix/ext/pg_repack.nix | 66 ++++++++++++ nix/ext/pg_stat_monitor.nix | 8 +- nix/ext/pg_tle.nix | 10 +- nix/ext/pgaudit.nix | 32 ++++++ nix/ext/pgjwt.nix | 31 ++++++ nix/ext/pgroonga.nix | 44 ++++++++ nix/ext/pgrouting.nix | 31 ++++++ nix/ext/pgsql-http.nix | 4 +- nix/ext/pgtap.nix | 33 ++++++ nix/ext/plpgsql-check.nix | 46 +++++++++ nix/ext/plv8.nix | 4 +- nix/ext/postgis.nix | 79 ++++++++++++++ nix/ext/rum.nix | 31 ++++++ nix/ext/supautils.nix | 4 +- nix/ext/timescaledb.nix | 44 ++++++++ nix/ext/wal2json.nix | 31 ++++++ nix/tools/sync-exts-versions.sh.in | 160 ++++++++++++++++++++++++++++- 23 files changed, 734 insertions(+), 45 deletions(-) create mode 100644 nix/ext/pg-safeupdate.nix create mode 100644 nix/ext/pg_cron.nix create mode 100644 nix/ext/pg_repack.nix create mode 100644 nix/ext/pgaudit.nix create mode 100644 nix/ext/pgjwt.nix create mode 100644 nix/ext/pgroonga.nix create mode 100644 nix/ext/pgrouting.nix create mode 100644 nix/ext/pgtap.nix create mode 100644 nix/ext/plpgsql-check.nix create mode 100644 nix/ext/postgis.nix create mode 100644 nix/ext/rum.nix create mode 100644 nix/ext/timescaledb.nix create mode 100644 nix/ext/wal2json.nix diff --git a/README.md b/README.md index f7b9eb5eb..ca794fc22 100644 --- a/README.md +++ b/README.md @@ -109,3 +109,12 @@ $ time packer build -timestamp-ui \ We are building the features of Firebase using enterprise-grade, open source products. We support existing communities wherever possible, and if the products don’t exist we build them and open source them ourselves. [![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase) + + +## Experimental Nix Packaging of resources + +There is a `/nix` folder in this repo, plus a `flake.nix` and `flake.lock` that facilitate using the Nix package management system to package supabase/postgres, and all of our extensions and wrappers. A user will need nix installed on their machine. As of 4/1/2024 the package set only builds on target machines (`x86_64-linux` and `aarch64-linux`), however work is under way to also support building and using directly on `aarch64-darwin` (macOs). As of 4/1/2024, versions of packages and extensions are synced from `/ansible/vars.yml` via a utility that can be run by executing `nix run .#sync-exts-versions` (you must have nix installed and be on the supported `x86_64-linux` and `aarch64-linux` for this command to work). The short term goal is to sync these versions as they are updated by our infrastructure and postgres teams, then to see the nix packaged versions build successfully in parallel over time, along with tests of the nix packaged versions passing. + +The supabase/postgres repo will continue to source it's dependencies from ansible for the short term, while we stabilize this nix build. + +Forthcoming PR's will include: integrating the nix work into our ansible/packer builds, building natively on aarch64-darwin (macOs), more testing diff --git a/flake.nix b/flake.nix index 0e6c53d0d..ec2ad0350 100644 --- a/flake.nix +++ b/flake.nix @@ -88,38 +88,13 @@ # listing their name. Anytime the version of nixpkgs is upgraded, these # may also bring in new versions of the extensions. psqlExtensions = [ - "postgis" - "pgrouting" - "pgtap" - "pg_cron" - "pgaudit" - "pgjwt" - "plpgsql_check" - "pg_safeupdate" - "wal2json" /* pljava */ - "rum" - "pg_repack" - "pgroonga" - "timescaledb" ]; #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue # with building timescaledb with the orioledb patched version of postgresql orioledbPsqlExtensions = [ - "postgis" - "pgrouting" - "pgtap" - "pg_cron" - "pgaudit" - "pgjwt" - "plpgsql_check" - "pg_safeupdate" - "wal2json" /* pljava */ - "rum" - "pg_repack" - "pgroonga" /*"timescaledb"*/ ]; @@ -133,6 +108,19 @@ # rollout new versions of these critical things easier without having to # go through the upstream release engineering process. ourExtensions = [ + ./nix/ext/rum.nix + ./nix/ext/timescaledb.nix + ./nix/ext/pgroonga.nix + ./nix/ext/wal2json.nix + ./nix/ext/pg_repack.nix + ./nix/ext/pg-safeupdate.nix + ./nix/ext/plpgsql-check.nix + ./nix/ext/pgjwt.nix + ./nix/ext/pgaudit.nix + ./nix/ext/postgis.nix + ./nix/ext/pgrouting.nix + ./nix/ext/pgtap.nix + ./nix/ext/pg_cron.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix ./nix/ext/pg_net.nix @@ -156,6 +144,7 @@ #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + postgis_override = pkgs.postgis_override; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other @@ -442,7 +431,7 @@ substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' chmod +x $out/bin/start-postgres-client ''; diff --git a/nix/ext/pg-safeupdate.nix b/nix/ext/pg-safeupdate.nix new file mode 100644 index 000000000..d24fab54a --- /dev/null +++ b/nix/ext/pg-safeupdate.nix @@ -0,0 +1,29 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg-safeupdate"; + version = "1.4"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "eradman"; + repo = pname; + rev = version; + hash = "sha256-1cyvVEC9MQGMr7Tg6EUbsVBrMc8ahdFS3+CmDkmAq4Y="; + }; + + installPhase = '' + install -D safeupdate${postgresql.dlSuffix} -t $out/lib + ''; + + meta = with lib; { + description = "A simple extension to PostgreSQL that requires criteria for UPDATE and DELETE"; + homepage = "https://github.com/eradman/pg-safeupdate"; + changelog = "https://github.com/eradman/pg-safeupdate/raw/${src.rev}/NEWS"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + broken = versionOlder postgresql.version "14"; + maintainers = with maintainers; [ samrose ]; + }; +} diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix new file mode 100644 index 000000000..468cc9834 --- /dev/null +++ b/nix/ext/pg_cron.nix @@ -0,0 +1,32 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg_cron"; + version = "1.6.2"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "citusdata"; + repo = pname; + rev = "v${version}"; + hash = "sha256-/dD1gX0+RRsBFIjSV9TVk+ppPw0Jrzssl+rRZ2qAp4w="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp *.so $out/lib + cp *.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Run Cron jobs through PostgreSQL"; + homepage = "https://github.com/citusdata/pg_cron"; + changelog = "https://github.com/citusdata/pg_cron/raw/v${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index a4497e68b..bdcd390d0 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.7.2"; + version = "0.8.0"; buildInputs = [ curl postgresql ]; @@ -10,9 +10,11 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-9Ki3fyinHTYrfckxAY0fCTlzJd9l+n7QRUV7mIWrqmc="; + hash = "sha256-ZPsRPWV1G3lMM2mT+H139Wvgoy8QnmeUbzEnGeDJmZA="; }; + env.NIX_CFLAGS_COMPILE = "-Wno-error"; + installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 12890d3b6..2d7d224ec 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -2,14 +2,14 @@ stdenv.mkDerivation rec { pname = "pg_plan_filter"; - version = "unstable-2021-09-23"; + version = "5081a7b5cb890876e67d8e7486b6a64c38c9a492"; buildInputs = [ postgresql ]; src = fetchFromGitHub { owner = "pgexperts"; repo = pname; - rev = "5081a7b5cb890876e67d8e7486b6a64c38c9a492"; + rev = "${version}"; hash = "sha256-YNeIfmccT/DtOrwDmpYFCuV2/P6k3Zj23VWBDkOh6sw="; }; diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix new file mode 100644 index 000000000..f3247378a --- /dev/null +++ b/nix/ext/pg_repack.nix @@ -0,0 +1,66 @@ +{ lib +, stdenv +, fetchFromGitHub +, openssl +, postgresql +, postgresqlTestHook +, readline +, testers +, zlib +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "pg_repack"; + version = "1.5.0"; + + buildInputs = postgresql.buildInputs ++ [ postgresql ]; + + src = fetchFromGitHub { + owner = "reorg"; + repo = "pg_repack"; + rev = "ver_${finalAttrs.version}"; + hash = "sha256-do80phyMxwcRIkYyUt9z02z7byNQhK+pbSaCUmzG+4c="; + }; + + installPhase = '' + install -D bin/pg_repack -t $out/bin/ + install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/ + install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension + ''; + + passthru.tests = { + version = testers.testVersion { + package = finalAttrs.finalPackage; + }; + extension = stdenv.mkDerivation { + name = "plpgsql-check-test"; + dontUnpack = true; + doCheck = true; + buildInputs = [ postgresqlTestHook ]; + nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ]; + postgresqlTestUserOptions = "LOGIN SUPERUSER"; + failureHook = "postgresqlStop"; + checkPhase = '' + runHook preCheck + psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;" + runHook postCheck + ''; + installPhase = "touch $out"; + }; + }; + + meta = with lib; { + description = "Reorganize tables in PostgreSQL databases with minimal locks"; + longDescription = '' + pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore + the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an + exclusive lock on the processed tables during processing. pg_repack is efficient to boot, + with performance comparable to using CLUSTER directly. + ''; + homepage = "https://github.com/reorg/pg_repack"; + license = licenses.bsd3; + maintainers = with maintainers; [ samrose ]; + inherit (postgresql.meta) platforms; + mainProgram = "pg_repack"; + }; +}) diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 954590607..bb1e1dc85 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -10,14 +10,14 @@ let # # this seems to all be cleaned up in version 2.0 of the extension, so ideally # we could upgrade to it later on and nuke this. - # sqlFilename = if lib.versionOlder postgresql.version "14" + # DEPRECATED sqlFilename = if lib.versionOlder postgresql.version "14" # then "pg_stat_monitor--1.0.13.sql.in" # else "pg_stat_monitor--1.0.14.sql.in"; in stdenv.mkDerivation rec { pname = "pg_stat_monitor"; - version = "2.0.4"; + version = "1.1.1"; buildInputs = [ postgresql ]; @@ -25,7 +25,7 @@ stdenv.mkDerivation rec { owner = "percona"; repo = pname; rev = "refs/tags/${version}"; - hash = "sha256-57Ji/KltIHNf81OxT0+4JIDqydST5RKMqrybNBZochg="; + hash = "sha256-S4N4Xnbkz57ue6f/eGjuRi64xT0NXjpMJiNNZnbbvbU="; }; makeFlags = [ "USE_PGXS=1" ]; @@ -33,7 +33,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp pg_stat_monitor--2.0.sql pg_stat_monitor--1.0--2.0.sql + cp pg_stat_monitor--1.0.15.sql.in pg_stat_monitor--1.0.sql cp *.so $out/lib cp *.sql $out/share/postgresql/extension diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 6188c7ce1..fd7b41a6c 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -1,21 +1,23 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, flex }: +{ lib, stdenv, fetchFromGitHub, postgresql, flex, openssl, libkrb5 }: stdenv.mkDerivation rec { pname = "pg_tle"; - version = "1.0.4"; + version = "1.3.2"; nativeBuildInputs = [ flex ]; - buildInputs = [ postgresql ]; + buildInputs = [ openssl postgresql libkrb5 ]; src = fetchFromGitHub { owner = "aws"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-W/7pLy/27VatCdzUh1NZ4K2FRMD1erfHiFV2eY2x2W0="; + hash = "sha256-g2up0hJ9y0yz6aKTRwNyJatnApMYz9hIp4lxT0I9bNs="; }; + makeFlags = [ "FLEX=flex" ]; + installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix new file mode 100644 index 000000000..c3df061ff --- /dev/null +++ b/nix/ext/pgaudit.nix @@ -0,0 +1,32 @@ +{ lib, stdenv, fetchFromGitHub, libkrb5, openssl, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgaudit"; + version = "1.7.0"; + + src = fetchFromGitHub { + owner = "pgaudit"; + repo = "pgaudit"; + rev = "${version}"; + hash = "sha256-8pShPr4HJaJQPjW1iPJIpj3CutTx8Tgr+rOqoXtgCcw="; + }; + + buildInputs = [ libkrb5 openssl postgresql ]; + + makeFlags = [ "USE_PGXS=1" ]; + + installPhase = '' + install -D -t $out/lib pgaudit${postgresql.dlSuffix} + install -D -t $out/share/postgresql/extension *.sql + install -D -t $out/share/postgresql/extension *.control + ''; + + meta = with lib; { + description = "Open Source PostgreSQL Audit Logging"; + homepage = "https://github.com/pgaudit/pgaudit"; + changelog = "https://github.com/pgaudit/pgaudit/releases/tag/${source.version}"; + maintainers = with maintainers; [ idontgetoutmuch ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pgjwt.nix b/nix/ext/pgjwt.nix new file mode 100644 index 000000000..2eb60f72e --- /dev/null +++ b/nix/ext/pgjwt.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, unstableGitUpdater }: + +stdenv.mkDerivation rec { + pname = "pgjwt"; + version = "9742dab1b2f297ad3811120db7b21451bca2d3c9"; + + src = fetchFromGitHub { + owner = "michelp"; + repo = "pgjwt"; + rev = "${version}"; + hash = "sha256-Hw3R9bMGDmh+dMzjmqZSy/rT4mX8cPU969OJiARFg10="; + }; + + dontBuild = true; + installPhase = '' + mkdir -p $out/share/postgresql/extension + cp pg*sql *.control $out/share/postgresql/extension + ''; + + passthru.updateScript = unstableGitUpdater { }; + + meta = with lib; { + description = "PostgreSQL implementation of JSON Web Tokens"; + longDescription = '' + sign() and verify() functions to create and verify JSON Web Tokens. + ''; + license = licenses.mit; + platforms = postgresql.meta.platforms; + maintainers = with maintainers; [samrose]; + }; +} diff --git a/nix/ext/pgroonga.nix b/nix/ext/pgroonga.nix new file mode 100644 index 000000000..bcd662727 --- /dev/null +++ b/nix/ext/pgroonga.nix @@ -0,0 +1,44 @@ +{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, groonga }: + +stdenv.mkDerivation rec { + pname = "pgroonga"; + version = "3.0.7"; + + src = fetchurl { + url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz"; + sha256 = "sha256-iF/zh4zDDpAw5fxW1WG8i2bfPt4VYsnYArwOoE/lwgM="; + }; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ postgresql msgpack-c groonga ]; + + makeFlags = [ + "HAVE_MSGPACK=1" + "MSGPACK_PACKAGE_NAME=msgpack-c" + ]; + + installPhase = '' + install -D pgroonga${postgresql.dlSuffix} -t $out/lib/ + install -D pgroonga.control -t $out/share/postgresql/extension + install -D data/pgroonga-*.sql -t $out/share/postgresql/extension + + install -D pgroonga_database${postgresql.dlSuffix} -t $out/lib/ + install -D pgroonga_database.control -t $out/share/postgresql/extension + install -D data/pgroonga_database-*.sql -t $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A PostgreSQL extension to use Groonga as the index"; + longDescription = '' + PGroonga is a PostgreSQL extension to use Groonga as the index. + PostgreSQL supports full text search against languages that use only alphabet and digit. + It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on. + You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL. + ''; + homepage = "https://pgroonga.github.io/"; + changelog = "https://github.com/pgroonga/pgroonga/releases/tag/${version}"; + license = licenses.postgresql; + platforms = postgresql.meta.platforms; + maintainers = with maintainers; [ samrose ]; + }; +} diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix new file mode 100644 index 000000000..b707454a8 --- /dev/null +++ b/nix/ext/pgrouting.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, perl, cmake, boost }: + +stdenv.mkDerivation rec { + pname = "pgrouting"; + version = "3.4.1"; + + nativeBuildInputs = [ cmake perl ]; + buildInputs = [ postgresql boost ]; + + src = fetchFromGitHub { + owner = "pgRouting"; + repo = pname; + rev = "v${version}"; + hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; + }; + + installPhase = '' + install -D lib/*.so -t $out/lib + install -D sql/pgrouting--${version}.sql -t $out/share/postgresql/extension + install -D sql/common/pgrouting.control -t $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A PostgreSQL/PostGIS extension that provides geospatial routing functionality"; + homepage = "https://pgrouting.org/"; + changelog = "https://github.com/pgRouting/pgrouting/releases/tag/v${version}"; + maintainers = [ steve-chavez samrose]; + platforms = postgresql.meta.platforms; + license = licenses.gpl2Plus; + }; +} diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 2edaa9d53..13da06fa3 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgsql-http"; - version = "1.6.0"; + version = "1.5.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "pramsey"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-CPHfx7vhWfxkXsoKTzyFuTt47BPMvzi/pi1leGcuD60="; + hash = "sha256-+N/CXm4arRgvhglanfvO0FNOBUWV5RL8mn/9FpNvcjY="; }; installPhase = '' diff --git a/nix/ext/pgtap.nix b/nix/ext/pgtap.nix new file mode 100644 index 000000000..c5a17c97f --- /dev/null +++ b/nix/ext/pgtap.nix @@ -0,0 +1,33 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, perl, perlPackages, which }: + +stdenv.mkDerivation rec { + pname = "pgtap"; + version = "1.2.0"; + + src = fetchFromGitHub { + owner = "theory"; + repo = "pgtap"; + rev = "v${version}"; + hash = "sha256-lb0PRffwo6J5a6Hqw1ggvn0cW7gPZ02OEcLPi9ineI8="; + }; + + nativeBuildInputs = [ postgresql perl perlPackages.TAPParserSourceHandlerpgTAP which ]; + + installPhase = '' + install -D {sql/pgtap--${version}.sql,pgtap.control} -t $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A unit testing framework for PostgreSQL"; + longDescription = '' + pgTAP is a unit testing framework for PostgreSQL written in PL/pgSQL and PL/SQL. + It includes a comprehensive collection of TAP-emitting assertion functions, + as well as the ability to integrate with other TAP-emitting test frameworks. + It can also be used in the xUnit testing style. + ''; + maintainers = with maintainers; [ samrose ]; + homepage = "https://pgtap.org"; + inherit (postgresql.meta) platforms; + license = licenses.mit; + }; +} diff --git a/nix/ext/plpgsql-check.nix b/nix/ext/plpgsql-check.nix new file mode 100644 index 000000000..c137e3027 --- /dev/null +++ b/nix/ext/plpgsql-check.nix @@ -0,0 +1,46 @@ +{ lib, stdenv, fetchFromGitHub, postgresql, postgresqlTestHook }: + +stdenv.mkDerivation rec { + pname = "plpgsql-check"; + version = "2.2.5"; + + src = fetchFromGitHub { + owner = "okbob"; + repo = "plpgsql_check"; + rev = "v${version}"; + hash = "sha256-47fuc0dnTWeu93P8zntsjT9L8SstN5p0WvnNtT4cHzM="; + }; + + buildInputs = [ postgresql ]; + + installPhase = '' + install -D -t $out/lib *${postgresql.dlSuffix} + install -D -t $out/share/postgresql/extension *.sql + install -D -t $out/share/postgresql/extension *.control + ''; + + passthru.tests.extension = stdenv.mkDerivation { + name = "plpgsql-check-test"; + dontUnpack = true; + doCheck = true; + buildInputs = [ postgresqlTestHook ]; + nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.plpgsql_check ])) ]; + postgresqlTestUserOptions = "LOGIN SUPERUSER"; + failureHook = "postgresqlStop"; + checkPhase = '' + runHook preCheck + psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION plpgsql_check;" + runHook postCheck + ''; + installPhase = "touch $out"; + }; + + meta = with lib; { + description = "Linter tool for language PL/pgSQL"; + homepage = "https://github.com/okbob/plpgsql_check"; + changelog = "https://github.com/okbob/plpgsql_check/releases/tag/v${version}"; + platforms = postgresql.meta.platforms; + license = licenses.mit; + maintainers = [ maintainers.marsam ]; + }; +} diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index bc92dc7ab..b3290b632 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -15,13 +15,13 @@ stdenv.mkDerivation (finalAttrs: { # plv8 latest is https://github.com/plv8/plv8/releases/tag/v3.2.2 # FIXME we need to increment this build toward 3.2.2 # 3.1.7 is the highest version that can be built with pg 16 - version = "3.1.7"; + version = "3.1.5"; src = fetchFromGitHub { owner = "plv8"; repo = "plv8"; rev = "v${finalAttrs.version}"; - hash = "sha256-kTID3Zo3YwNZUno8kdQE7ihtiddsIAZNuBN91IKgaC4="; + hash = "sha256-LodC2eQJSm5fLckrjm2RuejZhmOyQMJTv9b0iPCnzKQ="; }; patches = [ diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix new file mode 100644 index 000000000..02edf9c45 --- /dev/null +++ b/nix/ext/postgis.nix @@ -0,0 +1,79 @@ +{ fetchurl +, lib, stdenv +, perl +, libxml2 +, postgresql +, geos +, proj +, gdal +, json_c +, pkg-config +, file +, protobufc +, libiconv +, nixosTests +}: +stdenv.mkDerivation rec { + pname = "postgis"; + version = "3.3.2"; + + outputs = [ "out" "doc" ]; + + src = fetchurl { + url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; + sha256 = "sha256-miohnaAFoXMKOdGVmhx87GGbHvsAm2W+gP/CW60pkGg="; + }; + + buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc ] + ++ lib.optional stdenv.isDarwin libiconv; + nativeBuildInputs = [ perl pkg-config ] ++ lib.optional postgresql.jitSupport postgresql.llvm; + dontDisableStatic = true; + + # postgis config directory assumes /include /lib from the same root for json-c library + NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + + preConfigure = '' + sed -i 's@/usr/bin/file@${file}/bin/file@' configure + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev}" + + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin" + ''; + postConfigure = '' + sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; + s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/loader/Makefile"; + sed -i "s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/scripts/python/Makefile"; + mkdir -p $out/bin + + # postgis' build system assumes it is being installed to the same place as postgresql, and looks + # for the postgres binary relative to $PREFIX. We gently support this system using an illusion. + ln -s ${postgresql}/bin/postgres $out/bin/postgres + ''; + + # create aliases for all commands adding version information + postInstall = '' + # Teardown the illusory postgres used for building; see postConfigure. + rm $out/bin/postgres + + for prog in $out/bin/*; do # */ + ln -s $prog $prog-${version} + done + + mkdir -p $doc/share/doc/postgis + mv doc/* $doc/share/doc/postgis/ + ''; + + passthru.tests.postgis = nixosTests.postgis; + + meta = with lib; { + description = "Geographic Objects for PostgreSQL"; + homepage = "https://postgis.net/"; + changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; + license = licenses.gpl2; + maintainers = [ samrose ]; + inherit (postgresql.meta) platforms; + }; +} diff --git a/nix/ext/rum.nix b/nix/ext/rum.nix new file mode 100644 index 000000000..16bf106c5 --- /dev/null +++ b/nix/ext/rum.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "rum"; + version = "1.3.13"; + + src = fetchFromGitHub { + owner = "postgrespro"; + repo = "rum"; + rev = version; + hash = "sha256-yy2xeDnk3fENN+En0st4mv60nZlqPafIzwf68jwJ5fE="; + }; + + buildInputs = [ postgresql ]; + + makeFlags = [ "USE_PGXS=1" ]; + + installPhase = '' + install -D -t $out/lib *${postgresql.dlSuffix} + install -D -t $out/share/postgresql/extension *.control + install -D -t $out/share/postgresql/extension *.sql + ''; + + meta = with lib; { + description = "Full text search index method for PostgreSQL"; + homepage = "https://github.com/postgrespro/rum"; + license = licenses.postgresql; + platforms = postgresql.meta.platforms; + maintainers = with maintainers; [ samrose ]; + }; +} diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9e6d4d455..7b11ebf49 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "supautils"; - version = "1.8.0"; + version = "2.1.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-cQ294UNhlPtnqngGTVLYPMbGcqhqjFk5y6WBz6nCZhI="; + hash = "sha256-AKb8P+TvVRqw9pQhYka6rGzmcw5S1jpIZYH2kMisqyc="; }; installPhase = '' diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix new file mode 100644 index 000000000..d5bb60423 --- /dev/null +++ b/nix/ext/timescaledb.nix @@ -0,0 +1,44 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5, enableUnfree ? true }: + +stdenv.mkDerivation rec { + pname = "timescaledb${lib.optionalString (!enableUnfree) "-apache"}"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-gJViEWHtIczvIiQKuvvuwCfWJMxAYoBhCHhD75no6r0="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" ] + ++ lib.optionals (!enableUnfree) [ "-DAPACHE_ONLY=ON" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = with licenses; if enableUnfree then tsl else asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/ext/wal2json.nix b/nix/ext/wal2json.nix new file mode 100644 index 000000000..751eb64c0 --- /dev/null +++ b/nix/ext/wal2json.nix @@ -0,0 +1,31 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "wal2json"; + version = "2_5"; + + src = fetchFromGitHub { + owner = "eulerto"; + repo = "wal2json"; + rev = "wal2json_${builtins.replaceStrings ["."] ["_"] version}"; + hash = "sha256-Gpc9uDKrs/dmVSFgdgHM453+TaEnhRh9t0gDbSn8FUI="; + }; + + buildInputs = [ postgresql ]; + + makeFlags = [ "USE_PGXS=1" ]; + + installPhase = '' + install -D -t $out/lib *${postgresql.dlSuffix} + install -D -t $out/share/postgresql/extension sql/*.sql + ''; + + meta = with lib; { + description = "PostgreSQL JSON output plugin for changeset extraction"; + homepage = "https://github.com/eulerto/wal2json"; + changelog = "https://github.com/eulerto/wal2json/releases/tag/wal2json_${version}"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.bsd3; + }; +} diff --git a/nix/tools/sync-exts-versions.sh.in b/nix/tools/sync-exts-versions.sh.in index 322f2f40b..1b120e988 100644 --- a/nix/tools/sync-exts-versions.sh.in +++ b/nix/tools/sync-exts-versions.sh.in @@ -55,6 +55,55 @@ run_sync() { } +#for use where nix uses fetchurl +# instead of fetchFromGithub +fetchurl_source_url() { + local source_url=${1//\"/} # Remove double quotes + source_url=${source_url//\'/} # Remove single quotes + + # Check if the source URL is provided + if [ -z "$source_url" ]; then + echo "Usage: fetch_nix_url " + return 1 + fi + + echo "$source_url" + + # Run nix-prefetch-url command + local initial_hash=$($prefetchurl --type sha256 "$source_url" | cut -d ' ' -f 2) + #once we can bump up nix version, we can use nix hash convert --hash-algo sha256 + local final_hash=$($_nix hash to-sri --type sha256 $initial_hash) + echo "$final_hash" +} + +sync_version_fetchurl() { + + local package_name=$1 + local version="\"$2\"" + local hash="\"$3\"" + + + # Update the version and hash in the Nix expression + $editor $PWD/nix/ext/$package_name.nix version --inplace -v "$version" + $editor $PWD/nix/ext/$package_name.nix src.sha256 --inplace -v $hash +} + + +run_sync_fetchurl() { + local varname=$1 + local package_name=$2 + + version=$(echo $ansible_vars | $jq -r '.'$varname'') + echo "$key: $version" + url=$($_nix eval .#psql_15/exts/$package_name.src.url) + hash=$(fetchurl_source_url $url | tail -n 1) + $(sync_version_fetchurl $package_name $version $hash) + echo "synced $package_name to version $version with hash $hash" + + +} + +#for use on derivations that use cargoHash update_cargo_vendor_hash() { local package_name=$1 $editor $PWD/nix/ext/$package_name.nix cargoHash --inplace -v "" @@ -84,7 +133,6 @@ for key in $keys; do package_name="pg_hashids" run_sync $varname $package_name ;; - "hypopg_release") varname="hypopg_release" package_name="hypopg" @@ -96,17 +144,127 @@ for key in $keys; do run_sync $varname $package_name update_cargo_vendor_hash $package_name ;; + "pg_cron_release") + varname="pg_cron_release" + package_name="pg_cron" + run_sync $varname $package_name + ;; + "pgsql_http_release") + varname="pgsql_http_release" + package_name="pgsql-http" + run_sync $varname $package_name + ;; "pg_jsonschema_release") varname="pg_jsonschema_release" package_name="pg_jsonschema" run_sync $varname $package_name update_cargo_vendor_hash $package_name ;; + "pg_net_release") + varname="pg_net_release" + package_name="pg_net" + run_sync $varname $package_name + ;; + "pg_plan_filter_release") + varname="pg_plan_filter_release" + package_name="pg_plan_filter" + run_sync $varname $package_name + ;; + "pg_safeupdate_release") + varname="pg_safeupdate_release" + package_name="pg-safeupdate" + run_sync $varname $package_name + ;; + "pgsodium_release") + varname="pgsodium_release" + package_name="pgsodium" + run_sync $varname $package_name + ;; + "pg_repack_release") + varname="pg_repack_release" + package_name="pg_repack" + run_sync $varname $package_name + ;; + "pgrouting_release") + varname="pgrouting_release" + package_name="pgrouting" + run_sync $varname $package_name + ;; + "ptap_release") + varname="pgtap_release" + package_name="pgtap" + run_sync $varname $package_name + ;; + "pg_stat_monitor_release") + varname="pg_stat_monitor_release" + package_name="pg_stat_monitor" + run_sync $varname $package_name + ;; + "pg_tle_release") + varname="pg_tle_release" + package_name="pg_tle" + run_sync $varname $package_name + ;; + "pgaudit_release") + varname="pgaudit_release" + package_name="pgaudit" + run_sync $varname $package_name + ;; + "plpgsql_check_release") + varname="plpgsql_check_release" + package_name="plpgsql-check" + run_sync $varname $package_name + ;; "pgvector_release") varname="pgvector_release" package_name="pgvector" run_sync $varname $package_name ;; + "pgjwt_release") + varname="pgjwt_release" + package_name="pgjwt" + run_sync $varname $package_name + ;; + "plv8_release") + varname="plv8_release" + package_name="plv8" + run_sync $varname $package_name + ;; + "postgis_release") + varname="postgis_release" + package_name="postgis" + run_sync_fetchurl $varname $package_name + ;; + "pgroonga_release") + varname="pgroonga_release" + package_name="pgroonga" + run_sync_fetchurl $varname $package_name + ;; + "rum_release") + varname="rum_release" + package_name="rum" + run_sync $varname $package_name + ;; + "timescaledb_release") + varname="timescaledb_release" + package_name="timescaledb" + run_sync $varname $package_name + ;; + "supautils_release") + varname="supautils_release" + package_name="supautils" + run_sync $varname $package_name + ;; + "vault_release") + varname="vault_release" + package_name="vault" + run_sync $varname $package_name + ;; + "wal2json_release") + varname="wal2json_release" + package_name="wal2json" + run_sync $varname $package_name + ;; *) ;; esac