From 1cb583a896dee6e60258a75a48befa188fe75e3a Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 9 May 2025 17:34:27 -0700 Subject: [PATCH 01/42] Initial version of --dsql --- .gitmodules | 3 + README_DSQL.md | 236 ++++++++++++++++++++++++++++ aws-dsql-auth | 1 + build-dsql.sh | 79 ++++++++++ src/bin/psql/command.c | 7 +- src/bin/psql/common.h | 1 - src/bin/psql/help.c | 1 + src/bin/psql/settings.h | 1 + src/bin/psql/startup.c | 13 ++ src/interfaces/libpq/Makefile | 31 +++- src/interfaces/libpq/fe-connect.c | 38 ++++- src/interfaces/libpq/fe-dsql-auth.c | 187 ++++++++++++++++++++++ src/interfaces/libpq/fe-dsql-auth.h | 19 +++ src/interfaces/libpq/libpq-int.h | 1 + 14 files changed, 612 insertions(+), 6 deletions(-) create mode 100644 .gitmodules create mode 100644 README_DSQL.md create mode 160000 aws-dsql-auth create mode 100755 build-dsql.sh create mode 100644 src/interfaces/libpq/fe-dsql-auth.c create mode 100644 src/interfaces/libpq/fe-dsql-auth.h diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..0f3bdbe01c3a8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "aws-dsql-auth"] + path = aws-dsql-auth + url = https://github.com/marcbowes/aws-dsql-auth.git diff --git a/README_DSQL.md b/README_DSQL.md new file mode 100644 index 0000000000000..234dd9b201675 --- /dev/null +++ b/README_DSQL.md @@ -0,0 +1,236 @@ +# PostgreSQL Command Line Client (psql) with AWS DSQL Authentication + +This document describes how to build the PostgreSQL command line client (`psql`) with AWS DSQL Authentication support, which allows automatic token generation when connecting to AWS Database Services for PostgreSQL. + +## Overview of DSQL Authentication + +DSQL Authentication allows `psql` to automatically generate temporary authentication tokens when connecting to AWS Database Services for PostgreSQL. This integration offers several advantages: + +- No need to manually generate and paste tokens +- Automatic token regeneration if connections fail or tokens expire +- Simplified connection experience with a single `--dsql` flag + +## Prerequisites + +- C compiler (GCC or Clang) +- Make +- Readline library (for command line editing) +- OpenSSL development libraries (required for SSL support) +- AWS credentials configured in your environment (via ~/.aws/credentials, environment variables, etc.) +- AWS DSQL Auth library (located in `aws-dsql-auth` directory) + +## Build Steps + +### Quick Build with build-dsql.sh Script + +The easiest way to build PostgreSQL with DSQL Authentication support is to use the provided `build-dsql.sh` script: + +```bash +# Make the script executable if needed +chmod +x build-dsql.sh + +# Run the build script +./build-dsql.sh +``` + +This script automates the following tasks: +1. Initializes the aws-dsql-auth submodules if needed +2. Builds the AWS DSQL Auth library +3. Configures PostgreSQL with SSL support +4. Builds libpq and psql +5. Verifies SSL support +6. Displays usage instructions + +The script automatically detects your OS and sets appropriate configuration options. After running the script, you'll have a working psql binary with DSQL Authentication support. + +### Manual Build Steps + +If you prefer to build the components step-by-step, follow these instructions: + +#### 1. Install Required Dependencies + +##### On macOS: +```bash +brew install openssl readline +``` + +##### On Ubuntu/Debian: +```bash +sudo apt-get install libssl-dev libreadline-dev +``` + +##### On RHEL/Fedora: +```bash +sudo dnf install openssl-devel readline-devel +``` + +#### 2. Build the AWS DSQL Auth Library + +Initialize submodules (if not already done) and build the AWS DSQL Auth library: + +```bash +cd aws-dsql-auth +git submodule update --init --recursive +./build.sh +cd .. +``` + +#### 3. Configure PostgreSQL with SSL Support + +Configure the PostgreSQL source code with SSL support enabled: + +```bash +# macOS (with Homebrew) +./configure --with-openssl --with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib + +# Linux (typically SSL is found automatically) +./configure --with-openssl +``` + +Additional common options include: +- `--prefix=/path/to/install` - Set installation directory +- `--without-icu` - Disable ICU support if not available +- `--enable-tap-tests` - Enable TAP tests if you plan to run test suites + +#### 4. Build libpq (PostgreSQL Client Library) + +Build the PostgreSQL client library that psql depends on: + +```bash +make -C src/interfaces/libpq +``` + +#### 5. Build psql + +Build the psql command line utility: + +```bash +make -C src/bin/psql +``` + +#### 6. Verify SSL Support + +Ensure SSL support was built properly: + +```bash +src/bin/psql/psql --help | grep -i ssl +``` +You should see SSL-related options, or alternatively: +```bash +ldd src/bin/psql/psql | grep -i ssl # Linux +otool -L src/bin/psql/psql | grep -i ssl # macOS +``` + +### 7. Running psql + +To run the newly built psql with the locally built libpq: + +```bash +# On macOS +DYLD_LIBRARY_PATH=$(pwd)/src/interfaces/libpq ./src/bin/psql/psql [options] + +# On Linux +LD_LIBRARY_PATH=$(pwd)/src/interfaces/libpq ./src/bin/psql/psql [options] +``` + +Note: The AWS DSQL Auth library is statically linked into psql, so you only need to include the libpq library path. + +## Using DSQL Authentication + +To connect using DSQL authentication, add the `--dsql` flag to your `psql` command: + +```bash +./src/bin/psql/psql --dsql --host=your-dsql-endpoint.example.com --user=admin +``` + +Or, with the full connection string: + +```bash +./src/bin/psql/psql --dsql "dbname=postgres user=admin host=your-dsql-endpoint.example.com" +``` + +### How DSQL Authentication Works + +When the `--dsql` flag is provided: + +1. `psql` automatically sets `SSLMODE=require` to ensure a secure connection (SSL must be compiled in) +2. When connecting to a server, DSQL token generation is triggered +3. If the connection user is `admin`, the token is generated with admin privileges +4. A new token is generated for each connection attempt to ensure security +5. Tokens are set to expire after 5 seconds to enhance security +6. If the connection fails and reconnection is attempted, a new token is generated automatically + +### Environment Variables + +- `PGSSLMODE`: If not already set and `--dsql` is used, it will be set to `require` +- `AWS_REGION`: Optional, if not set the region will be inferred from the hostname +- `PGSSLROOTCERT`: Optional, path to SSL root certificate for verification + +## Testing + +To run tests specifically for libpq and psql: + +```bash +# Test libpq +make -C src/interfaces/libpq check + +# Test psql +make -C src/bin/psql check +``` + +Note: For comprehensive testing, configure with `--enable-tap-tests` before running these commands. + +## Optional: Installation + +If you want to install psql and libpq: + +```bash +make -C src/bin/psql install +make -C src/interfaces/libpq install +make -C src/include install # For header files +``` + +## Troubleshooting + +### SSL-Related Issues + +1. If you see "sslmode value 'require' invalid when SSL support is not compiled in": + - Ensure you configured with `--with-openssl` + - Check that OpenSSL development files are properly installed + - Rebuild from scratch after configuring with SSL support + +2. If you experience SSL connection issues: + - You can specify certificate verification settings with environment variables: + ```bash + PGSSLMODE=verify-ca PGSSLROOTCERT=/path/to/rootcert.pem ./src/bin/psql/psql --dsql ... + ``` + +### General Build Issues + +If you encounter library dependency issues: + +1. Ensure the library path is correctly set to point to your built libpq and aws-dsql-auth libraries +2. Check for missing dependencies with `ldd` (Linux) or `otool -L` (macOS) +3. Verify that the configure step completed without errors + +### DSQL Authentication Issues + +If you encounter issues with DSQL authentication: + +1. Ensure your AWS credentials are correctly configured and have appropriate permissions +2. Check that you're using the correct endpoint hostname +3. Verify the `aws-dsql-auth` library is correctly built +4. Try with explicit AWS credentials via environment variables: + ```bash + AWS_ACCESS_KEY_ID=your_key AWS_SECRET_ACCESS_KEY=your_secret ./src/bin/psql/psql --dsql ... + ``` + +## Notes + +- This builds only the command line client, not the PostgreSQL server +- The built psql can connect to any compatible PostgreSQL server +- Command line editing requires readline support +- SSL support is required for DSQL Authentication to work properly +- DSQL Authentication is primarily intended for use with AWS Database Services for PostgreSQL +- Token generation happens automatically and is transparent to the user +- Currently, admin token generation is only supported for the `admin` user diff --git a/aws-dsql-auth b/aws-dsql-auth new file mode 160000 index 0000000000000..1581ca076ecc2 --- /dev/null +++ b/aws-dsql-auth @@ -0,0 +1 @@ +Subproject commit 1581ca076ecc261f8b6acf5b7743ef105ce74b2f diff --git a/build-dsql.sh b/build-dsql.sh new file mode 100755 index 0000000000000..160b87f23f1d2 --- /dev/null +++ b/build-dsql.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -e + +echo "Building PostgreSQL with DSQL Authentication support" +echo "===================================================" + +# Determine OS type for library path configuration +if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Detected macOS" + LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" + OS_SPECIFIC_CONFIG="--with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib" +else + echo "Detected Linux/Unix system" + LIBRARY_PATH_VAR="LD_LIBRARY_PATH" + OS_SPECIFIC_CONFIG="" +fi + +# Step 1: Initialize and build aws-dsql-auth +echo "Step 1: Setting up AWS DSQL Auth library..." + +# Check if aws-dsql-auth submodules are initialized +if [ ! -d "aws-dsql-auth/aws-sdk/aws-c-common/.git" ]; then + echo " Initializing aws-dsql-auth submodules..." + cd aws-dsql-auth + git submodule update --init --recursive + cd .. +else + echo " aws-dsql-auth submodules already initialized." +fi + +# Build aws-dsql-auth +echo " Building aws-dsql-auth library..." +cd aws-dsql-auth +./build.sh +cd .. +echo " AWS DSQL Auth library built successfully!" + +# Step 2: Configure PostgreSQL with SSL support +echo "Step 2: Configuring PostgreSQL with SSL support..." +if [ ! -f "config.status" ]; then + echo " Running configure with SSL support..." + ./configure --with-openssl $OS_SPECIFIC_CONFIG +else + echo " PostgreSQL already configured. If you need to reconfigure, run './configure --with-openssl $OS_SPECIFIC_CONFIG' manually." +fi + +# Step 3: Build libpq (PostgreSQL client library) +echo "Step 3: Building libpq..." +make -C src/interfaces/libpq +echo " libpq built successfully!" + +# Step 4: Build psql +echo "Step 4: Building psql..." +make -C src/bin/psql +echo " psql built successfully!" + +# Verify SSL support +echo "Verifying SSL support in psql binary..." +if [[ "$OSTYPE" == "darwin"* ]]; then + otool -L src/bin/psql/psql | grep -i ssl +else + ldd src/bin/psql/psql | grep -i ssl +fi + +# Final instructions +echo "" +echo "Build completed successfully!" +echo "" +echo "To run psql with DSQL authentication, use the following command:" +echo "" +echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" +echo " ./src/bin/psql/psql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "" +echo "Or with connection string format:" +echo "" +echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" +echo " ./src/bin/psql/psql --dsql \"dbname=postgres user=admin host=your-dsql-endpoint.example.com\"" +echo "" +echo "Note: You need to have AWS credentials configured in your environment for DSQL authentication to work." diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 81a5ba844ba0f..856584f87b9b6 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -4035,6 +4035,11 @@ do_connect(enum trivalue reuse_previous_specification, success = false; } + /* When in DSQL mode, we never reuse passwords (as tokens may have expired). */ + if (pset.dsql) { + keep_password = false; + } + /* * If the user asked to be prompted for a password, ask for one now. If * not, use the password from the old connection, provided the username @@ -4045,7 +4050,7 @@ do_connect(enum trivalue reuse_previous_specification, * the postmaster's log. But libpq offers no API that would let us obtain * a password and then continue with the first connection attempt. */ - if (pset.getPassword == TRI_YES && success) + if (pset.getPassword == TRI_YES && success && !pset.dsql) { bool canceled = false; diff --git a/src/bin/psql/common.h b/src/bin/psql/common.h index 7f1a23de1e82d..d587604a7bcd3 100644 --- a/src/bin/psql/common.h +++ b/src/bin/psql/common.h @@ -14,7 +14,6 @@ #include "fe_utils/print.h" #include "fe_utils/psqlscan.h" #include "libpq-fe.h" - extern bool openQueryOutputFile(const char *fname, FILE **fout, bool *is_pipe); extern bool setQFout(const char *fname); diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index 403b51325a72c..ec79a9df51ee0 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -113,6 +113,7 @@ usage(unsigned short int pager) HELP0(" -U, --username=USERNAME database user name\n"); HELP0(" -w, --no-password never prompt for password\n"); HELP0(" -W, --password force password prompt (should happen automatically)\n"); + HELP0(" --dsql enable Aurora DSQL mode\n"); HELP0("\nFor more information, type \"\\?\" (for internal commands) or \"\\help\" (for SQL\n" "commands) from within psql, or consult the psql section in the PostgreSQL\n" diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h index fd82303f776c4..75997b5f068f9 100644 --- a/src/bin/psql/settings.h +++ b/src/bin/psql/settings.h @@ -169,6 +169,7 @@ typedef struct _psqlSettings bool singlestep; bool hide_compression; bool hide_tableam; + bool dsql; /* --dsql command line option */ int fetch_count; int histsize; int ignoreeof; diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 249b6aa516902..a167556231316 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -186,6 +186,7 @@ main(int argc, char *argv[]) pset.notty = (!isatty(fileno(stdin)) || !isatty(fileno(stdout))); pset.getPassword = TRI_DEFAULT; + pset.dsql = false; /* Initialize dsql flag to false */ EstablishVariableSpace(); @@ -211,6 +212,13 @@ main(int argc, char *argv[]) SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", "0"); parse_psql_options(argc, argv, &options); + + if (pset.dsql) + { + setenv("PGSSLMODE", "require", 0); + setenv("PGDSQL", "1", 1); + pset.getPassword = TRI_NO; + } /* * If no action was specified and we're in non-interactive mode, treat it @@ -274,6 +282,7 @@ main(int argc, char *argv[]) values[7] = NULL; new_pass = false; + pset.db = PQconnectdbParams(keywords, values, true); free(keywords); free(values); @@ -525,6 +534,7 @@ parse_psql_options(int argc, char *argv[], struct adhoc_opts *options) {"no-psqlrc", no_argument, NULL, 'X'}, {"help", optional_argument, NULL, 1}, {"csv", no_argument, NULL, 2}, + {"dsql", no_argument, NULL, 3}, {NULL, 0, NULL, 0} }; @@ -718,6 +728,9 @@ parse_psql_options(int argc, char *argv[], struct adhoc_opts *options) case 2: pset.popt.topt.format = PRINT_CSV; break; + case 3: + pset.dsql = true; + break; default: unknown_option: /* getopt_long already emitted a complaint */ diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index c6fe5fec7f64d..2492a59c80907 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -24,7 +24,16 @@ NAME= pq SO_MAJOR_VERSION= 5 SO_MINOR_VERSION= $(MAJORVERSION) -override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port +override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port \ + -I$(top_srcdir)/aws-dsql-auth/include \ + -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-auth/include \ + -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-common/include \ + -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-io/include \ + -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-sdkutils/include \ + -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common/generated/include \ + -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth/generated/include \ + -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io/generated/include \ + -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils/generated/include ifneq ($(PORTNAME), win32) override CFLAGS += $(PTHREAD_CFLAGS) endif @@ -44,7 +53,8 @@ OBJS = \ legacy-pqsignal.o \ libpq-events.o \ pqexpbuffer.o \ - fe-auth.o + fe-auth.o \ + fe-dsql-auth.o # File shared across all SSL implementations supported. ifneq ($(with_ssl),no) @@ -85,9 +95,22 @@ endif # matter.) Note that we filter out -lpgcommon and -lpgport from LIBS and # instead link with -lpgcommon_shlib and -lpgport_shlib, to get object files # that are built correctly for use in a shlib. -SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib +SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib \ + -L$(top_srcdir)/aws-dsql-auth/build -laws-dsql-auth \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common -laws-c-common \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth -laws-c-auth \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io -laws-c-io \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils -laws-c-sdkutils \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-cal -laws-c-cal \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-http -laws-c-http \ + -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-compression -laws-c-compression ifneq ($(PORTNAME), win32) SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) + +# Add macOS-specific frameworks +ifeq ($(PORTNAME),darwin) +SHLIB_LINK += -framework CoreFoundation -framework Security -framework Network +endif else SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif @@ -161,6 +184,7 @@ install: all installdirs install-lib $(INSTALL_DATA) $(srcdir)/libpq-events.h '$(DESTDIR)$(includedir)' $(INSTALL_DATA) $(srcdir)/libpq-int.h '$(DESTDIR)$(includedir_internal)' $(INSTALL_DATA) $(srcdir)/fe-auth-sasl.h '$(DESTDIR)$(includedir_internal)' + $(INSTALL_DATA) $(srcdir)/fe-dsql-auth.h '$(DESTDIR)$(includedir_internal)' $(INSTALL_DATA) $(srcdir)/pqexpbuffer.h '$(DESTDIR)$(includedir_internal)' $(INSTALL_DATA) $(srcdir)/pg_service.conf.sample '$(DESTDIR)$(datadir)/pg_service.conf.sample' @@ -183,6 +207,7 @@ uninstall: uninstall-lib rm -f '$(DESTDIR)$(includedir)/libpq-events.h' rm -f '$(DESTDIR)$(includedir_internal)/libpq-int.h' rm -f '$(DESTDIR)$(includedir_internal)/fe-auth-sasl.h' + rm -f '$(DESTDIR)$(includedir_internal)/fe-dsql-auth.h' rm -f '$(DESTDIR)$(includedir_internal)/pqexpbuffer.h' rm -f '$(DESTDIR)$(datadir)/pg_service.conf.sample' diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 430c0fa44428a..8b0bf0960f5ef 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -23,10 +23,12 @@ #include #include "common/base64.h" +#include "common/logging.h" #include "common/ip.h" #include "common/link-canary.h" #include "common/scram-common.h" #include "common/string.h" +#include "fe-dsql-auth.h" #include "fe-auth.h" #include "fe-auth-oauth.h" #include "libpq-fe.h" @@ -879,6 +881,12 @@ PQconnectStartParams(const char *const *keywords, if (conn == NULL) return NULL; + if (strcmp(getenv("PGDSQL"), "1") == 0) { + conn->is_dsql = true; + conn->sslmode = strdup("direct"); + conn->auth_required = true; + } + /* * Parse the conninfo arrays */ @@ -1418,11 +1426,39 @@ pqConnectOptions2(PGconn *conn) goto oom_error; } + /* + * In DSQL mode, generate auth tokens. + */ + if (conn->is_dsql) + { + for (i = 0; i < conn->nconnhost; i++) + { + bool is_admin; + char *token; + char *err_msg = NULL; + const char *pwhost = conn->connhost[i].host; + if (pwhost == NULL || pwhost[0] == '\0') + pwhost = conn->connhost[i].hostaddr; + + is_admin = strcmp("admin", conn->pguser) == 0; + token = generate_dsql_token(pwhost, is_admin, &err_msg); + if (!token) + { + libpq_append_conn_error(conn, "DSQL token generation failed for host=%s: %s", + pwhost, err_msg ? err_msg : "unknown error"); + if (err_msg) + free(err_msg); + } + else + conn->connhost[i].password = token; + } + } + /* * If password was not given, try to look it up in password file. Note * that the result might be different for each host/port pair. */ - if (conn->pgpass == NULL || conn->pgpass[0] == '\0') + if ((conn->pgpass == NULL || conn->pgpass[0] == '\0') && !conn->is_dsql) { /* If password file wasn't specified, use ~/PGPASSFILE */ if (conn->pgpassfile == NULL || conn->pgpassfile[0] == '\0') diff --git a/src/interfaces/libpq/fe-dsql-auth.c b/src/interfaces/libpq/fe-dsql-auth.c new file mode 100644 index 0000000000000..217e19c1c9c4f --- /dev/null +++ b/src/interfaces/libpq/fe-dsql-auth.c @@ -0,0 +1,187 @@ +/* + * fe-dsql-auth.c + * + * Support for AWS DSQL authentication token generation + * + * Copyright (c) 2025 PostgreSQL Global Development Group + */ +#include "postgres_fe.h" + +#include "fe-dsql-auth.h" +#include "libpq-int.h" + +#include +#include +#include +#include + +/* Include AWS DSQL Auth library functions */ +#include +#include +#include +#include +#include +#include + +static bool aws_libs_initialized = false; + +/* + * This function can be called to clean up AWS library resources + */ +static void +_dsql_auth_cleanup(void) +{ + if (aws_libs_initialized) + { + aws_sdkutils_library_clean_up(); + aws_auth_library_clean_up(); + aws_io_library_clean_up(); + aws_common_library_clean_up(); + aws_libs_initialized = false; + } +} + +/* + * Clean up DSQL authentication resources + */ +void +dsql_auth_cleanup(void) +{ + _dsql_auth_cleanup(); +} + +/* + * Initialize AWS libraries if not already initialized + */ +static void +initialize_aws_libs(void) +{ + if (!aws_libs_initialized) + { + struct aws_allocator *allocator = aws_default_allocator(); + aws_common_library_init(allocator); + aws_io_library_init(allocator); + aws_auth_library_init(allocator); + aws_sdkutils_library_init(allocator); + aws_libs_initialized = true; + + /* Note: We cannot use atexit() in libpq as it's not allowed to call exit-related functions. + * The cleanup will be handled by explicit calls at application shutdown or by the OS. + */ + } +} + +/* + * Generate a DSQL authentication token for the specified endpoint. + * Uses the AWS DSQL auth library to generate a real token. + * Returns a newly allocated string containing the token. + */ +char * +generate_dsql_token(const char *endpoint, bool admin, char **err_msg) +{ + struct aws_allocator *allocator; + struct aws_dsql_auth_config auth_config; + struct aws_dsql_auth_token auth_token = {0}; + struct aws_string *aws_region = NULL; + struct aws_credentials_provider *credentials_provider = NULL; + struct aws_credentials_provider_chain_default_options credentials_options; + char *token = NULL; + int aws_error; + const char *env_region; + const char *token_str; + + /* Initialize AWS libraries */ + initialize_aws_libs(); + + allocator = aws_default_allocator(); + + /* Initialize DSQL auth config */ + if (aws_dsql_auth_config_init(&auth_config) != AWS_OP_SUCCESS) { + if (err_msg) + *err_msg = strdup("Failed to initialize DSQL auth config"); + goto cleanup; + } + + /* Set hostname */ + aws_dsql_auth_config_set_hostname(&auth_config, endpoint); + + /* Try to get region from environment variable first */ + env_region = getenv("AWS_REGION"); + if (env_region != NULL && env_region[0] != '\0') + { + aws_region = aws_string_new_from_c_str(allocator, env_region); + if (!aws_region) { + if (err_msg) + *err_msg = strdup("Failed to create region string from AWS_REGION"); + goto cleanup; + } + } + else + { + /* Try to infer region from hostname */ + if (aws_dsql_auth_config_infer_region(allocator, &auth_config, &aws_region) != AWS_OP_SUCCESS || + aws_region == NULL) + { + if (err_msg) + *err_msg = strdup("Failed to infer AWS region from hostname. Please set AWS_REGION environment variable."); + goto cleanup; + } + } + aws_dsql_auth_config_set_region(&auth_config, aws_region); + + /* Create default credentials provider */ + AWS_ZERO_STRUCT(credentials_options); + + credentials_provider = aws_credentials_provider_new_chain_default(allocator, &credentials_options); + if (!credentials_provider) { + aws_error = aws_last_error(); + if (err_msg) + *err_msg = strdup(aws_error_str(aws_error)); + goto cleanup; + } + + /* Set credentials provider */ + aws_dsql_auth_config_set_credentials_provider(&auth_config, credentials_provider); + + /* Set expiration time to 5 seconds for shorter token lifetime */ + aws_dsql_auth_config_set_expires_in(&auth_config, 5); /* 5 seconds */ + + /* Generate the token */ + AWS_ZERO_STRUCT(auth_token); + if (aws_dsql_auth_token_generate(&auth_config, admin, allocator, &auth_token) != AWS_OP_SUCCESS) + { + aws_error = aws_last_error(); + if (err_msg) + *err_msg = strdup(aws_error_str(aws_error)); + goto cleanup; + } + + /* Get the token string */ + token_str = aws_dsql_auth_token_get_str(&auth_token); + if (token_str) + { + token = strdup(token_str); + /* Token generation successful */ + } + else + { + if (err_msg) + *err_msg = strdup("Failed to get token string"); + } + +cleanup: + aws_dsql_auth_token_clean_up(&auth_token); + aws_dsql_auth_config_clean_up(&auth_config); + + if (credentials_provider) + { + aws_credentials_provider_release(credentials_provider); + } + + if (aws_region) + { + aws_string_destroy(aws_region); + } + + return token; +} diff --git a/src/interfaces/libpq/fe-dsql-auth.h b/src/interfaces/libpq/fe-dsql-auth.h new file mode 100644 index 0000000000000..23c65c2f0cd5a --- /dev/null +++ b/src/interfaces/libpq/fe-dsql-auth.h @@ -0,0 +1,19 @@ +/* + * fe-dsql-auth.h + * + * Support for AWS DSQL authentication token generation + * + * Copyright (c) 2025 PostgreSQL Global Development Group + */ +#ifndef FE_DSQL_AUTH_H +#define FE_DSQL_AUTH_H + +#include + +/* Generate a DSQL authentication token for the specified endpoint */ +char *generate_dsql_token(const char *endpoint, bool admin, char **err_msg); + +/* Clean up DSQL authentication resources */ +void dsql_auth_cleanup(void); + +#endif /* FE_DSQL_AUTH_H */ diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index a6cfd7f5c9d83..44e4f9f7fd4ea 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -520,6 +520,7 @@ struct pg_conn char current_auth_response; /* used by pqTraceOutputMessage to * know which auth response we're * sending */ + bool is_dsql; /* when true, auth tokens are auto-generated */ /* Callbacks for external async authentication */ PostgresPollingStatusType (*async_auth) (PGconn *conn); From c21bd20ca841c756b67b6b4bcfeca427febd5f68 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 13 May 2025 11:12:24 -0700 Subject: [PATCH 02/42] Fix icu4c on macOS --- build-dsql.sh | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/build-dsql.sh b/build-dsql.sh index 160b87f23f1d2..cc5b3afa73820 100755 --- a/build-dsql.sh +++ b/build-dsql.sh @@ -8,7 +8,24 @@ echo "===================================================" if [[ "$OSTYPE" == "darwin"* ]]; then echo "Detected macOS" LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" + + # Base configuration with OpenSSL OS_SPECIFIC_CONFIG="--with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib" + + # Check for ICU4C in Homebrew + if [ -d "/opt/homebrew/opt/icu4c" ]; then + echo " Detected Homebrew ICU4C installation" + # Use the Homebrew-maintained symlink to the current version + export PKG_CONFIG_PATH="/opt/homebrew/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" + echo " Added ICU4C to build configuration" + elif [ -d "/usr/local/opt/icu4c" ]; then + # For Intel Macs with Homebrew installed in /usr/local + echo " Detected Homebrew ICU4C installation in /usr/local" + export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH" + echo " Added ICU4C to build configuration" + else + echo " Warning: Homebrew ICU4C not detected, configure may fail if ICU is required" + fi else echo "Detected Linux/Unix system" LIBRARY_PATH_VAR="LD_LIBRARY_PATH" @@ -38,10 +55,10 @@ echo " AWS DSQL Auth library built successfully!" # Step 2: Configure PostgreSQL with SSL support echo "Step 2: Configuring PostgreSQL with SSL support..." if [ ! -f "config.status" ]; then - echo " Running configure with SSL support..." - ./configure --with-openssl $OS_SPECIFIC_CONFIG + echo " Running configure with SSL and DSQL support..." + ./configure --with-openssl --with-dsql $OS_SPECIFIC_CONFIG else - echo " PostgreSQL already configured. If you need to reconfigure, run './configure --with-openssl $OS_SPECIFIC_CONFIG' manually." + echo " PostgreSQL already configured. If you need to reconfigure, run './configure --with-openssl --with-dsql $OS_SPECIFIC_CONFIG' manually." fi # Step 3: Build libpq (PostgreSQL client library) From 0c16ff000f749060cb30c6b83b8536f78567683b Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 13 May 2025 13:48:39 -0700 Subject: [PATCH 03/42] Don't need openssl --- build-dsql.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/build-dsql.sh b/build-dsql.sh index cc5b3afa73820..c770fd8f64e16 100755 --- a/build-dsql.sh +++ b/build-dsql.sh @@ -55,10 +55,10 @@ echo " AWS DSQL Auth library built successfully!" # Step 2: Configure PostgreSQL with SSL support echo "Step 2: Configuring PostgreSQL with SSL support..." if [ ! -f "config.status" ]; then - echo " Running configure with SSL and DSQL support..." - ./configure --with-openssl --with-dsql $OS_SPECIFIC_CONFIG + echo " Running configure ..." + ./configure $OS_SPECIFIC_CONFIG else - echo " PostgreSQL already configured. If you need to reconfigure, run './configure --with-openssl --with-dsql $OS_SPECIFIC_CONFIG' manually." + echo " PostgreSQL already configured. If you need to reconfigure, run './configure $OS_SPECIFIC_CONFIG' manually." fi # Step 3: Build libpq (PostgreSQL client library) @@ -71,14 +71,6 @@ echo "Step 4: Building psql..." make -C src/bin/psql echo " psql built successfully!" -# Verify SSL support -echo "Verifying SSL support in psql binary..." -if [[ "$OSTYPE" == "darwin"* ]]; then - otool -L src/bin/psql/psql | grep -i ssl -else - ldd src/bin/psql/psql | grep -i ssl -fi - # Final instructions echo "" echo "Build completed successfully!" From d79d0005c1929b679a32a341bd2085b7abac8a91 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 13 May 2025 14:27:32 -0700 Subject: [PATCH 04/42] Packaging - mac only (for now) --- .github/workflows/build-dsql-macos-arm.yml | 51 ++++++++++++ README_DSQL.md | 92 ++++++++++++++++++++++ build-dsql.sh => scripts/build-dsql.sh | 0 scripts/package.sh | 80 +++++++++++++++++++ scripts/test-packaging.sh | 53 +++++++++++++ src/bin/psql/startup.c | 4 + 6 files changed, 280 insertions(+) create mode 100644 .github/workflows/build-dsql-macos-arm.yml rename build-dsql.sh => scripts/build-dsql.sh (100%) create mode 100755 scripts/package.sh create mode 100755 scripts/test-packaging.sh diff --git a/.github/workflows/build-dsql-macos-arm.yml b/.github/workflows/build-dsql-macos-arm.yml new file mode 100644 index 0000000000000..0ed43c6fac590 --- /dev/null +++ b/.github/workflows/build-dsql-macos-arm.yml @@ -0,0 +1,51 @@ +name: Build and Package DSQL for macOS ARM + +on: + workflow_dispatch: # Allow manual triggering + push: + tags: + - "v*" # Run on version tags + +jobs: + build: + runs-on: macos-latest + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: "recursive" + + - name: Set up macOS + run: | + brew update + brew install openssl@3 icu4c zip + + - name: Build DSQL + run: | + chmod +x scripts/build-dsql.sh + ./scripts/build-dsql.sh + + - name: Package for Distribution + run: | + chmod +x scripts/package.sh + ./scripts/package.sh + + - name: Test Package + run: | + chmod +x scripts/test-packaging.sh + ./scripts/test-packaging.sh --test-only + + - name: Upload Build Artifact + uses: actions/upload-artifact@v4 + with: + name: postgres-dsql + path: postgres-dsql.zip + + - name: Release + if: startsWith(github.ref, 'refs/tags/') + uses: softprops/action-gh-release@v1 + with: + files: postgres-dsql.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README_DSQL.md b/README_DSQL.md index 234dd9b201675..2d315d363eb40 100644 --- a/README_DSQL.md +++ b/README_DSQL.md @@ -234,3 +234,95 @@ If you encounter issues with DSQL authentication: - DSQL Authentication is primarily intended for use with AWS Database Services for PostgreSQL - Token generation happens automatically and is transparent to the user - Currently, admin token generation is only supported for the `admin` user + +## Packaging and Distribution + +PostgreSQL DSQL client can be packaged into a distributable format that can be easily downloaded from GitHub Releases. + +### Overview of the Packaging System + +The packaging system: + +- Takes the built `psql` binary and `libpq` library +- Copies them into a structured directory called `postgres-dsql` +- Renames `psql` to `pdsql` +- Sets up the proper relative library paths so `pdsql` can find `libpq` +- Creates a ZIP archive for distribution + +### Directory Structure + +The packaged distribution has the following structure: + +``` +postgres-dsql/ +├── bin/ +│ └── pdsql # Renamed psql binary +└── lib/ + └── libpq.5.dylib # PostgreSQL client library +``` + +### GitHub Actions Workflow + +The GitHub Actions workflow automates the build and packaging process, and is triggered: + +- Manually via workflow_dispatch +- On creation of version tags (v*) + +To create a new release: +1. Tag a commit with a version number: `git tag v1.0.0` +2. Push the tag: `git push origin v1.0.0` + +The workflow will: +1. Build PostgreSQL with DSQL support +2. Package the binaries +3. Test the package +4. Upload the package as a GitHub release artifact + +### Local Development and Testing + +#### Testing the Packaging Locally + +To test the packaging process locally: + +```bash +# Build if needed and package +./scripts/test-packaging.sh +``` + +This script will: +1. Build DSQL if not already built +2. Package the build into postgres-dsql.zip +3. Extract and test the binary by running `dpsql --version` + +#### Manual Testing + +You can also test manually: + +```bash +# Build DSQL +./scripts/build-dsql.sh + +# Package the build +./scripts/package.sh + +# Test the packaged binary +unzip -o postgres-dsql.zip -d /tmp +/tmp/postgres-dsql/bin/pdsql --version +``` + +### Using the Packaged Binary + +After downloading and extracting the package: + +```bash +unzip postgres-dsql.zip +cd postgres-dsql +./bin/pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres +``` + +Note: The renamed binary `pdsql` automatically enables DSQL authentication mode, so the `--dsql` flag is not necessary. + +### Current Limitations + +- This packaging solution currently only supports ARM macOS +- The package assumes the necessary AWS credentials are available in the environment diff --git a/build-dsql.sh b/scripts/build-dsql.sh similarity index 100% rename from build-dsql.sh rename to scripts/build-dsql.sh diff --git a/scripts/package.sh b/scripts/package.sh new file mode 100755 index 0000000000000..282556865e2c9 --- /dev/null +++ b/scripts/package.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -e + +# Package PostgreSQL DSQL client for distribution +# This script creates a standalone distribution with psql (renamed to pdsql) +# and libpq that can be used without additional dependencies + +echo "Packaging PostgreSQL DSQL client" +echo "================================" + +# Path setup +ROOT_DIR=$(pwd) +DIST_NAME="postgres-dsql" +DIST_DIR="$ROOT_DIR/$DIST_NAME" +SRC_BIN="$ROOT_DIR/src/bin/psql/psql" +SRC_LIB="$ROOT_DIR/src/interfaces/libpq/libpq.5.dylib" +BINARY_NAME="pdsql" + +# Check if we're on macOS +if [[ "$OSTYPE" != "darwin"* ]]; then + echo "This script is currently only designed for macOS" + exit 1 +fi + +# Check if the build artifacts exist +if [ ! -f "$SRC_BIN" ]; then + echo "Error: psql binary not found at $SRC_BIN" + echo "Please run scripts/build-dsql.sh first" + exit 1 +fi + +if [ ! -f "$SRC_LIB" ]; then + echo "Error: libpq library not found at $SRC_LIB" + echo "Please run scripts/build-dsql.sh first" + exit 1 +fi + +# Clean any previous packaging attempts +if [ -d "$DIST_DIR" ]; then + echo "Cleaning previous packaging directory..." + rm -rf "$DIST_DIR" +fi + +# Create directory structure +mkdir -p "$DIST_DIR/bin" +mkdir -p "$DIST_DIR/lib" + +# Copy binaries and libraries +echo "Copying psql to $DIST_DIR/bin/$BINARY_NAME" +cp "$SRC_BIN" "$DIST_DIR/bin/$BINARY_NAME" + +echo "Copying libpq to $DIST_DIR/lib/" +cp "$SRC_LIB" "$DIST_DIR/lib/" +cp "$ROOT_DIR/src/interfaces/libpq/libpq.dylib" "$DIST_DIR/lib/" 2>/dev/null || true + +# Set up correct library paths in the binary +echo "Updating library paths in $BINARY_NAME binary..." +LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq | awk '{print $1}') +install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$BINARY_NAME" + +# Fix library itself to refer to itself by relative path +install_name_tool -id "@loader_path/libpq.5.dylib" "$DIST_DIR/lib/libpq.5.dylib" + +# Verify the changes +echo "Verifying library path changes:" +otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq +otool -L "$DIST_DIR/lib/libpq.5.dylib" | grep libpq + +# Create a ZIP archive +echo "Creating ZIP archive..." +ZIP_NAME="${DIST_NAME}.zip" +rm -f "$ZIP_NAME" +(cd "$ROOT_DIR" && zip -r "$ZIP_NAME" "$DIST_NAME") + +echo "Package created at $ROOT_DIR/$ZIP_NAME" +echo "Done!" + +# For testing, you can: +# unzip -o postgres-dsql.zip -d /tmp +# /tmp/postgres-dsql/bin/pdsql --version diff --git a/scripts/test-packaging.sh b/scripts/test-packaging.sh new file mode 100755 index 0000000000000..61ed2d1100a0d --- /dev/null +++ b/scripts/test-packaging.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +# Test script to verify local packaging +echo "Testing local packaging workflow" +echo "===============================" + +# Check if we should skip building and packaging +SKIP_BUILD_PACKAGE=0 +if [ "$1" == "--test-only" ]; then + SKIP_BUILD_PACKAGE=1 +fi + +if [ $SKIP_BUILD_PACKAGE -eq 0 ]; then + # Build DSQL if needed + if [ ! -f "src/bin/psql/psql" ]; then + echo "Building DSQL first..." + ./scripts/build-dsql.sh + else + echo "DSQL already built, skipping build step" + fi + + # Package the build + echo "Packaging DSQL..." + ./scripts/package.sh +else + echo "Skipping build and package steps, testing only..." +fi + +# Test the packaged binary +echo "Testing packaged binary..." +if [ -f "postgres-dsql.zip" ]; then + # Clean any previous test + rm -rf /tmp/postgres-dsql + + # Extract and test + unzip -o postgres-dsql.zip -d /tmp + + echo "Running pdsql --version to verify:" + /tmp/postgres-dsql/bin/pdsql --version + + if [ $? -eq 0 ]; then + echo "✅ Test passed! Package is working correctly." + else + echo "❌ Test failed! Please check the package." + exit 1 + fi +else + echo "❌ Package file not found!" + exit 1 +fi + +echo "All tests completed successfully." diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index a167556231316..e3300224f5791 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -211,6 +211,10 @@ main(int argc, char *argv[]) SetVariable(pset.vars, "PIPELINE_COMMAND_COUNT", "0"); SetVariable(pset.vars, "PIPELINE_RESULT_COUNT", "0"); + /* Automatically enable dsql mode if the program is named "pdsql" */ + if (strcmp(pset.progname, "pdsql") == 0) + pset.dsql = true; + parse_psql_options(argc, argv, &options); if (pset.dsql) From f261410ff46270837a17864694c64a8a044156f2 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 13 May 2025 15:14:08 -0700 Subject: [PATCH 05/42] Install script --- scripts/build-dsql.sh | 2 +- scripts/install.sh | 94 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100755 scripts/install.sh diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index c770fd8f64e16..ed41971ee5d62 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -10,7 +10,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" # Base configuration with OpenSSL - OS_SPECIFIC_CONFIG="--with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib" + OS_SPECIFIC_CONFIG="--with-ssl=openssl --with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib" # Check for ICU4C in Homebrew if [ -d "/opt/homebrew/opt/icu4c" ]; then diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000000000..b77f0a051581b --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -e + +REPO="marcbowes/postgres" +DEFAULT_INSTALL_PATH="$HOME/.local" + +# Function to display error messages and exit +error_exit() { + echo "Error: $1" >&2 + exit 1 +} + +# Check if running on macOS +if [[ "$OSTYPE" != "darwin"* ]]; then + error_exit "This script is only supported on macOS systems." +fi + +# Check if running on ARM architecture +if [[ "$(uname -m)" != "arm64" ]]; then + error_exit "This script is only intended for ARM-based macOS (Apple Silicon) systems." +fi + +echo "PostgreSQL DSQL ARM macOS Installer" +echo "===================================" + +# Get latest release information using GitHub API +echo "Fetching latest release information..." +RELEASE_INFO=$(curl -s "https://api.github.com/repos/$REPO/releases/latest") +if [[ -z "$RELEASE_INFO" || "$RELEASE_INFO" == *"Not Found"* ]]; then + error_exit "Could not fetch release information. Check your internet connection." +fi + +# Extract download URL for the zip file +DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o '"browser_download_url": *"[^"]*postgres-dsql.zip"' | cut -d'"' -f4) +if [[ -z "$DOWNLOAD_URL" ]]; then + error_exit "No postgres-dsql.zip found in the latest release." +fi + +TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) +echo "Latest release found: $TAG_NAME" + +# Prompt for installation path +read -p "Enter installation path [$DEFAULT_INSTALL_PATH]: " INSTALL_PATH +INSTALL_PATH=${INSTALL_PATH:-$DEFAULT_INSTALL_PATH} + +# Create directories if they don't exist +mkdir -p "$INSTALL_PATH/bin" +mkdir -p "$INSTALL_PATH/lib" + +# Download the release +echo "Downloading release from $DOWNLOAD_URL..." +TEMP_DIR=$(mktemp -d) +curl -L "$DOWNLOAD_URL" -o "$TEMP_DIR/postgres-dsql.zip" + +# Extract the release +echo "Extracting files to $INSTALL_PATH..." +unzip -o "$TEMP_DIR/postgres-dsql.zip" -d "$TEMP_DIR" + +# Copy files to install location +cp -r "$TEMP_DIR/postgres-dsql/bin/"* "$INSTALL_PATH/bin/" +cp -r "$TEMP_DIR/postgres-dsql/lib/"* "$INSTALL_PATH/lib/" + +# Clean up temp files +rm -rf "$TEMP_DIR" + +# Make the binary executable +chmod +x "$INSTALL_PATH/bin/pdsql" + +echo "Installation completed successfully!" +echo "PostgreSQL DSQL (pdsql) installed to: $INSTALL_PATH/bin/pdsql" + +# Check if installation path is in PATH +if [[ ":$PATH:" != *":$INSTALL_PATH/bin:"* ]]; then + echo "" + echo "NOTICE: Your PATH environment variable doesn't contain $INSTALL_PATH/bin" + echo "To add it to your PATH, add the following line to your $HOME/.bashrc or $HOME/.zshrc file:" + echo "" + echo " export PATH=\"$INSTALL_PATH/bin:\$PATH\"" + echo "" + echo "Then, reload your shell configuration by running:" + echo "" + if [[ "$SHELL" == *"zsh"* ]]; then + echo " source $HOME/.zshrc" + else + echo " source $HOME/.bashrc" + fi +fi + +echo "" +echo "To verify installation, run:" +echo "" +echo " pdsql --version" +echo "" +echo "If the command is not found, ensure your PATH is set correctly as described above." From 9f023da621a2cb4169c4991bee913d7e0d82d3d4 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 13 May 2025 16:03:38 -0700 Subject: [PATCH 06/42] Remove install prompt --- scripts/install.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index b77f0a051581b..cf124cbd87607 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2,7 +2,7 @@ set -e REPO="marcbowes/postgres" -DEFAULT_INSTALL_PATH="$HOME/.local" +INSTALL_PATH="$HOME/.local" # Function to display error messages and exit error_exit() { @@ -39,10 +39,6 @@ fi TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) echo "Latest release found: $TAG_NAME" -# Prompt for installation path -read -p "Enter installation path [$DEFAULT_INSTALL_PATH]: " INSTALL_PATH -INSTALL_PATH=${INSTALL_PATH:-$DEFAULT_INSTALL_PATH} - # Create directories if they don't exist mkdir -p "$INSTALL_PATH/bin" mkdir -p "$INSTALL_PATH/lib" From eea3c4da5deb16a100104568acadbe17eb0688f6 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Thu, 29 May 2025 12:06:43 -0700 Subject: [PATCH 07/42] readline --- .github/workflows/build-dsql-macos-arm.yml | 2 +- scripts/build-dsql.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-dsql-macos-arm.yml b/.github/workflows/build-dsql-macos-arm.yml index 0ed43c6fac590..b11ecf4d00528 100644 --- a/.github/workflows/build-dsql-macos-arm.yml +++ b/.github/workflows/build-dsql-macos-arm.yml @@ -19,7 +19,7 @@ jobs: - name: Set up macOS run: | brew update - brew install openssl@3 icu4c zip + brew install openssl@3 icu4c zip readline - name: Build DSQL run: | diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index ed41971ee5d62..b910489138991 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -9,8 +9,8 @@ if [[ "$OSTYPE" == "darwin"* ]]; then echo "Detected macOS" LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" - # Base configuration with OpenSSL - OS_SPECIFIC_CONFIG="--with-ssl=openssl --with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib" + # Base configuration with OpenSSL and readline + OS_SPECIFIC_CONFIG="--with-ssl=openssl --with-includes=/opt/homebrew/opt/openssl/include:/opt/homebrew/opt/readline/include --with-libraries=/opt/homebrew/opt/openssl/lib:/opt/homebrew/opt/readline/lib" # Check for ICU4C in Homebrew if [ -d "/opt/homebrew/opt/icu4c" ]; then From cab0e033bf969a6af3f8020f03f16cc87839905f Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Tue, 3 Jun 2025 14:41:29 -0700 Subject: [PATCH 08/42] Bump dsql auth --- aws-dsql-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-dsql-auth b/aws-dsql-auth index 1581ca076ecc2..25f7cba5998e2 160000 --- a/aws-dsql-auth +++ b/aws-dsql-auth @@ -1 +1 @@ -Subproject commit 1581ca076ecc261f8b6acf5b7743ef105ce74b2f +Subproject commit 25f7cba5998e2f257a4daa57e76bdf75aeee8bc9 From 3c3e72257db00725822ca4af673932a2cc5e1b0a Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Wed, 18 Jun 2025 21:51:31 -0700 Subject: [PATCH 09/42] Static dsql-auth --- aws-dsql-auth | 2 +- src/interfaces/libpq/Makefile | 34 +++++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/aws-dsql-auth b/aws-dsql-auth index 25f7cba5998e2..b1394444b53c8 160000 --- a/aws-dsql-auth +++ b/aws-dsql-auth @@ -1 +1 @@ -Subproject commit 25f7cba5998e2f257a4daa57e76bdf75aeee8bc9 +Subproject commit b1394444b53c85fb49d8ca03b24e66e2b7c5077d diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 2492a59c80907..2f37d89327e91 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -90,20 +90,32 @@ OBJS += \ endif +# AWS DSQL Auth static libraries - these are embedded into fe-dsql-auth.o only +AWS_DSQL_AUTH_LIBS = \ + $(top_srcdir)/aws-dsql-auth/build/libaws-dsql-auth.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth/libaws-c-auth.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-http/libaws-c-http.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io/libaws-c-io.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-cal/libaws-c-cal.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils/libaws-c-sdkutils.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-compression/libaws-c-compression.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common/libaws-c-common.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-lc/ssl/libssl.a \ + $(top_srcdir)/aws-dsql-auth/build/aws-lc/crypto/libcrypto.a \ + $(top_srcdir)/aws-dsql-auth/build/lib/libs2n.a + +# Custom rule for fe-dsql-auth.o that includes AWS libraries +fe-dsql-auth.o: fe-dsql-auth.c $(AWS_DSQL_AUTH_LIBS) + $(CC) $(CFLAGS) $(CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o + $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) + rm -f fe-dsql-auth-temp.o + # Add libraries that libpq depends (or might depend) on into the # shared library link. (The order in which you list them here doesn't # matter.) Note that we filter out -lpgcommon and -lpgport from LIBS and # instead link with -lpgcommon_shlib and -lpgport_shlib, to get object files # that are built correctly for use in a shlib. -SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib \ - -L$(top_srcdir)/aws-dsql-auth/build -laws-dsql-auth \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common -laws-c-common \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth -laws-c-auth \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io -laws-c-io \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils -laws-c-sdkutils \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-cal -laws-c-cal \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-http -laws-c-http \ - -L$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-compression -laws-c-compression +SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) @@ -160,7 +172,7 @@ $(stlib): $(OBJS_STATIC) libpq-refs-stamp: $(shlib) ifneq ($(enable_coverage), yes) ifeq (,$(filter solaris,$(PORTNAME))) - @if nm -A -u $< 2>/dev/null | grep -v -e __cxa_atexit -e __tsan_func_exit | grep exit; then \ + @if nm -A -u $< 2>/dev/null | grep -v -e __cxa_atexit -e __tsan_func_exit | grep -v s2n_disable_atexit | grep exit; then \ echo 'libpq must not be calling any function which invokes exit'; exit 1; \ fi endif @@ -214,6 +226,6 @@ uninstall: uninstall-lib clean distclean: clean-lib $(MAKE) -C test $@ rm -rf tmp_check - rm -f $(OBJS) $(OBJS_SHLIB) $(OBJS_STATIC) pthread.h libpq-refs-stamp + rm -f $(OBJS) $(OBJS_SHLIB) $(OBJS_STATIC) pthread.h libpq-refs-stamp fe-dsql-auth-temp.o # Might be left over from a Win32 client-only build rm -f pg_config_paths.h From 774f932fe9d6ad3c45ae54b6a1aba5527b44bbe7 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 12:17:25 -0700 Subject: [PATCH 10/42] It builds on linux --- aws-dsql-auth | 2 +- scripts/build-dsql.sh | 16 ++++++++++------ src/interfaces/libpq/Makefile | 31 +++++-------------------------- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/aws-dsql-auth b/aws-dsql-auth index b1394444b53c8..86a5f98749538 160000 --- a/aws-dsql-auth +++ b/aws-dsql-auth @@ -1 +1 @@ -Subproject commit b1394444b53c85fb49d8ca03b24e66e2b7c5077d +Subproject commit 86a5f9874953866409095f3f2391596ec51dd1a9 diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index b910489138991..7015b53bef140 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -45,12 +45,16 @@ else echo " aws-dsql-auth submodules already initialized." fi -# Build aws-dsql-auth -echo " Building aws-dsql-auth library..." -cd aws-dsql-auth -./build.sh -cd .. -echo " AWS DSQL Auth library built successfully!" +if [ ! -f "aws-dsql-auth/build/aws-dsql-auth/libaws-dsql-auth.a" ]; then + # Build aws-dsql-auth + echo " Building aws-dsql-auth library..." + cd aws-dsql-auth + ./build.sh + cd .. + echo " AWS DSQL Auth library built successfully!" +else + echo " aws-dsql-auth already built." +fi # Step 2: Configure PostgreSQL with SSL support echo "Step 2: Configuring PostgreSQL with SSL support..." diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 2f37d89327e91..8877713f34c07 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -24,16 +24,7 @@ NAME= pq SO_MAJOR_VERSION= 5 SO_MINOR_VERSION= $(MAJORVERSION) -override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port \ - -I$(top_srcdir)/aws-dsql-auth/include \ - -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-auth/include \ - -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-common/include \ - -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-io/include \ - -I$(top_srcdir)/aws-dsql-auth/aws-sdk/aws-c-sdkutils/include \ - -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common/generated/include \ - -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth/generated/include \ - -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io/generated/include \ - -I$(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils/generated/include +override CPPFLAGS := -I$(srcdir) $(CPPFLAGS) -I$(top_builddir)/src/port -I$(top_srcdir)/src/port ifneq ($(PORTNAME), win32) override CFLAGS += $(PTHREAD_CFLAGS) endif @@ -89,24 +80,12 @@ OBJS += \ win32.o endif - -# AWS DSQL Auth static libraries - these are embedded into fe-dsql-auth.o only -AWS_DSQL_AUTH_LIBS = \ - $(top_srcdir)/aws-dsql-auth/build/libaws-dsql-auth.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-auth/libaws-c-auth.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-http/libaws-c-http.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-io/libaws-c-io.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-cal/libaws-c-cal.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-sdkutils/libaws-c-sdkutils.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-compression/libaws-c-compression.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-sdk/aws-c-common/libaws-c-common.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-lc/ssl/libssl.a \ - $(top_srcdir)/aws-dsql-auth/build/aws-lc/crypto/libcrypto.a \ - $(top_srcdir)/aws-dsql-auth/build/lib/libs2n.a +AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include)) +AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a') # Custom rule for fe-dsql-auth.o that includes AWS libraries -fe-dsql-auth.o: fe-dsql-auth.c $(AWS_DSQL_AUTH_LIBS) - $(CC) $(CFLAGS) $(CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o +fe-dsql-auth.o: fe-dsql-auth.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) rm -f fe-dsql-auth-temp.o From 0c45380c97a1adb5477a6f2b849de757fbaa81d9 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:19:47 -0700 Subject: [PATCH 11/42] Build on ubuntu --- ...uild-dsql-macos-arm.yml => build-dsql.yml} | 30 ++++++-- scripts/build-dsql.sh | 48 +++++++++++- scripts/package.sh | 75 ++++++++++++++----- 3 files changed, 129 insertions(+), 24 deletions(-) rename .github/workflows/{build-dsql-macos-arm.yml => build-dsql.yml} (60%) diff --git a/.github/workflows/build-dsql-macos-arm.yml b/.github/workflows/build-dsql.yml similarity index 60% rename from .github/workflows/build-dsql-macos-arm.yml rename to .github/workflows/build-dsql.yml index b11ecf4d00528..d301f5b498bb2 100644 --- a/.github/workflows/build-dsql-macos-arm.yml +++ b/.github/workflows/build-dsql.yml @@ -1,14 +1,18 @@ -name: Build and Package DSQL for macOS ARM +name: Build and Package DSQL on: workflow_dispatch: # Allow manual triggering push: - tags: - - "v*" # Run on version tags + branches: ["*"] # Run on all branch pushes + pull_request: + branches: ["*"] # Run on all pull requests jobs: build: - runs-on: macos-latest + strategy: + matrix: + os: [macos-latest, ubuntu-22.04] + runs-on: ${{ matrix.os }} steps: - name: Check out repository @@ -17,10 +21,26 @@ jobs: submodules: "recursive" - name: Set up macOS + if: runner.os == 'macOS' run: | brew update brew install openssl@3 icu4c zip readline + - name: Set up Ubuntu + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + flex \ + bison \ + libreadline-dev \ + zlib1g-dev \ + libssl-dev \ + libicu-dev \ + build-essential \ + cmake \ + zip + - name: Build DSQL run: | chmod +x scripts/build-dsql.sh @@ -39,7 +59,7 @@ jobs: - name: Upload Build Artifact uses: actions/upload-artifact@v4 with: - name: postgres-dsql + name: postgres-dsql-${{ runner.os }}-${{ runner.arch }} path: postgres-dsql.zip - name: Release diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 7015b53bef140..5a4bd05cb3741 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -26,10 +26,54 @@ if [[ "$OSTYPE" == "darwin"* ]]; then else echo " Warning: Homebrew ICU4C not detected, configure may fail if ICU is required" fi +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "Detected Linux system" + LIBRARY_PATH_VAR="LD_LIBRARY_PATH" + + # Base configuration for Linux with system packages + OS_SPECIFIC_CONFIG="--with-ssl=openssl --with-icu" + + # Check if we have the required development packages + echo " Checking for required development packages..." + + # Check for essential build tools and libraries + MISSING_PACKAGES="" + + if ! dpkg -l | grep -q "libssl-dev\|openssl-dev" 2>/dev/null && ! rpm -qa | grep -q "openssl-devel" 2>/dev/null; then + MISSING_PACKAGES="$MISSING_PACKAGES libssl-dev" + fi + + if ! dpkg -l | grep -q "libreadline-dev\|readline-dev" 2>/dev/null && ! rpm -qa | grep -q "readline-devel" 2>/dev/null; then + MISSING_PACKAGES="$MISSING_PACKAGES libreadline-dev" + fi + + if ! dpkg -l | grep -q "zlib1g-dev\|zlib-dev" 2>/dev/null && ! rpm -qa | grep -q "zlib-devel" 2>/dev/null; then + MISSING_PACKAGES="$MISSING_PACKAGES zlib1g-dev" + fi + + if ! dpkg -l | grep -q "libicu-dev\|icu-dev" 2>/dev/null && ! rpm -qa | grep -q "libicu-devel" 2>/dev/null; then + MISSING_PACKAGES="$MISSING_PACKAGES libicu-dev" + fi + + if ! command -v flex >/dev/null 2>&1; then + MISSING_PACKAGES="$MISSING_PACKAGES flex" + fi + + if ! command -v bison >/dev/null 2>&1; then + MISSING_PACKAGES="$MISSING_PACKAGES bison" + fi + + if [ -n "$MISSING_PACKAGES" ]; then + echo " Warning: Some required packages may be missing: $MISSING_PACKAGES" + echo " On Ubuntu/Debian, install with: sudo apt-get install $MISSING_PACKAGES" + echo " On RHEL/CentOS, install equivalent packages with yum/dnf" + else + echo " Required development packages appear to be installed" + fi else - echo "Detected Linux/Unix system" + echo "Detected Unix system (assuming Linux-like)" LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - OS_SPECIFIC_CONFIG="" + OS_SPECIFIC_CONFIG="--with-ssl=openssl" fi # Step 1: Initialize and build aws-dsql-auth diff --git a/scripts/package.sh b/scripts/package.sh index 282556865e2c9..bbd5fb57f28e4 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -13,12 +13,22 @@ ROOT_DIR=$(pwd) DIST_NAME="postgres-dsql" DIST_DIR="$ROOT_DIR/$DIST_NAME" SRC_BIN="$ROOT_DIR/src/bin/psql/psql" -SRC_LIB="$ROOT_DIR/src/interfaces/libpq/libpq.5.dylib" BINARY_NAME="pdsql" -# Check if we're on macOS -if [[ "$OSTYPE" != "darwin"* ]]; then - echo "This script is currently only designed for macOS" +# Detect OS and set appropriate library paths +if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Detected macOS" + SRC_LIB="$ROOT_DIR/src/interfaces/libpq/libpq.5.dylib" + LIB_EXT="dylib" + PLATFORM="macos" +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "Detected Linux" + SRC_LIB="$ROOT_DIR/src/interfaces/libpq/libpq.so.5" + LIB_EXT="so" + PLATFORM="linux" +else + echo "Error: Unsupported operating system: $OSTYPE" + echo "This script supports macOS and Linux only" exit 1 fi @@ -51,20 +61,48 @@ cp "$SRC_BIN" "$DIST_DIR/bin/$BINARY_NAME" echo "Copying libpq to $DIST_DIR/lib/" cp "$SRC_LIB" "$DIST_DIR/lib/" -cp "$ROOT_DIR/src/interfaces/libpq/libpq.dylib" "$DIST_DIR/lib/" 2>/dev/null || true -# Set up correct library paths in the binary -echo "Updating library paths in $BINARY_NAME binary..." -LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq | awk '{print $1}') -install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$BINARY_NAME" - -# Fix library itself to refer to itself by relative path -install_name_tool -id "@loader_path/libpq.5.dylib" "$DIST_DIR/lib/libpq.5.dylib" - -# Verify the changes -echo "Verifying library path changes:" -otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq -otool -L "$DIST_DIR/lib/libpq.5.dylib" | grep libpq +# Handle platform-specific library setup +if [[ "$PLATFORM" == "macos" ]]; then + # Copy additional dylib if it exists + cp "$ROOT_DIR/src/interfaces/libpq/libpq.dylib" "$DIST_DIR/lib/" 2>/dev/null || true + + # Set up correct library paths in the binary + echo "Updating library paths in $BINARY_NAME binary..." + LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq | awk '{print $1}') + install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$BINARY_NAME" + + # Fix library itself to refer to itself by relative path + install_name_tool -id "@loader_path/libpq.5.dylib" "$DIST_DIR/lib/libpq.5.dylib" + + # Verify the changes + echo "Verifying library path changes:" + otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq + otool -L "$DIST_DIR/lib/libpq.5.dylib" | grep libpq + +elif [[ "$PLATFORM" == "linux" ]]; then + # Copy additional .so files if they exist + cp "$ROOT_DIR/src/interfaces/libpq/libpq.so" "$DIST_DIR/lib/" 2>/dev/null || true + + # Set up RPATH for the binary to find libraries in ../lib + echo "Setting RPATH for $BINARY_NAME binary..." + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" 2>/dev/null || { + echo "Warning: patchelf not available. Installing patchelf..." + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update && sudo apt-get install -y patchelf + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y patchelf + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" + else + echo "Warning: Could not install patchelf. The binary may not find libraries correctly." + fi + } + + # Verify the changes + echo "Verifying RPATH changes:" + ldd "$DIST_DIR/bin/$BINARY_NAME" | grep libpq || echo "libpq dependency check complete" +fi # Create a ZIP archive echo "Creating ZIP archive..." @@ -78,3 +116,6 @@ echo "Done!" # For testing, you can: # unzip -o postgres-dsql.zip -d /tmp # /tmp/postgres-dsql/bin/pdsql --version +# +# On Linux, you may also need to ensure the library path is set: +# LD_LIBRARY_PATH=/tmp/postgres-dsql/lib /tmp/postgres-dsql/bin/pdsql --version From a16d60cfed9bb3e14219906bc41690c67101a055 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:23:23 -0700 Subject: [PATCH 12/42] Rpm --- .github/workflows/build-dsql.yml | 15 +++- scripts/package.sh | 117 +++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index d301f5b498bb2..664d1943dbcd0 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -39,7 +39,9 @@ jobs: libicu-dev \ build-essential \ cmake \ - zip + zip \ + rpm \ + patchelf - name: Build DSQL run: | @@ -62,10 +64,19 @@ jobs: name: postgres-dsql-${{ runner.os }}-${{ runner.arch }} path: postgres-dsql.zip + - name: Upload RPM Artifact + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: postgres-dsql-rpm-${{ runner.arch }} + path: "*.rpm" + - name: Release if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v1 with: - files: postgres-dsql.zip + files: | + postgres-dsql.zip + *.rpm env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/package.sh b/scripts/package.sh index bbd5fb57f28e4..293fc6bc53a6c 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -111,6 +111,119 @@ rm -f "$ZIP_NAME" (cd "$ROOT_DIR" && zip -r "$ZIP_NAME" "$DIST_NAME") echo "Package created at $ROOT_DIR/$ZIP_NAME" + +# Create RPM package for Linux +if [[ "$PLATFORM" == "linux" ]]; then + echo "Creating RPM package..." + + # Create RPM build directory structure + RPM_BUILD_DIR="$ROOT_DIR/rpmbuild" + mkdir -p "$RPM_BUILD_DIR"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + + # Create spec file + SPEC_FILE="$RPM_BUILD_DIR/SPECS/postgres-dsql.spec" + cat > "$SPEC_FILE" << 'EOF' +Name: postgres-dsql +Version: 1.0.0 +Release: 1%{?dist} +Summary: PostgreSQL DSQL client (pdsql) - AWS DSQL authentication enabled psql +License: PostgreSQL +URL: https://github.com/your-org/postgres-dsql +BuildArch: x86_64 + +%description +PostgreSQL DSQL client provides pdsql, a PostgreSQL client with AWS DSQL +authentication support. This package installs alongside existing PostgreSQL +installations without conflicts by using different binary and library names. + +%prep +# No prep needed - files are already prepared + +%build +# No build needed - binaries are already built + +%install +mkdir -p %{buildroot}/opt/postgres-dsql/bin +mkdir -p %{buildroot}/opt/postgres-dsql/lib +mkdir -p %{buildroot}/usr/bin + +# Install binaries and libraries to /opt to avoid conflicts +cp %{_sourcedir}/bin/pdsql %{buildroot}/opt/postgres-dsql/bin/ +cp %{_sourcedir}/lib/* %{buildroot}/opt/postgres-dsql/lib/ + +# Create symlink in /usr/bin for easy access +ln -s /opt/postgres-dsql/bin/pdsql %{buildroot}/usr/bin/pdsql + +%files +/opt/postgres-dsql/bin/pdsql +/opt/postgres-dsql/lib/* +/usr/bin/pdsql + +%post +echo "PostgreSQL DSQL client installed successfully!" +echo "Use 'pdsql' command to connect to AWS DSQL databases." +echo "Example: pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" + +%changelog +* $(date +'%a %b %d %Y') Build System - 1.0.0-1 +- Initial RPM package for PostgreSQL DSQL client +EOF + + # Copy files to SOURCES directory with the structure expected by the spec + mkdir -p "$RPM_BUILD_DIR/SOURCES/bin" + mkdir -p "$RPM_BUILD_DIR/SOURCES/lib" + cp "$DIST_DIR/bin/pdsql" "$RPM_BUILD_DIR/SOURCES/bin/" + cp "$DIST_DIR/lib"/* "$RPM_BUILD_DIR/SOURCES/lib/" + + # Build the RPM + echo "Building RPM package..." + if command -v rpmbuild >/dev/null 2>&1; then + rpmbuild --define "_topdir $RPM_BUILD_DIR" -bb "$SPEC_FILE" + + # Find and copy the generated RPM + RPM_FILE=$(find "$RPM_BUILD_DIR/RPMS" -name "*.rpm" | head -1) + if [ -n "$RPM_FILE" ]; then + cp "$RPM_FILE" "$ROOT_DIR/" + RPM_NAME=$(basename "$RPM_FILE") + echo "RPM package created at $ROOT_DIR/$RPM_NAME" + else + echo "Warning: RPM file not found after build" + fi + else + echo "Warning: rpmbuild not available. Installing rpm-build..." + if command -v yum >/dev/null 2>&1; then + sudo yum install -y rpm-build + elif command -v dnf >/dev/null 2>&1; then + sudo dnf install -y rpm-build + elif command -v apt-get >/dev/null 2>&1; then + sudo apt-get update && sudo apt-get install -y rpm + else + echo "Error: Could not install rpm-build. RPM package not created." + echo "Please install rpm-build manually and re-run this script." + fi + + # Retry RPM build if rpmbuild is now available + if command -v rpmbuild >/dev/null 2>&1; then + rpmbuild --define "_topdir $RPM_BUILD_DIR" -bb "$SPEC_FILE" + RPM_FILE=$(find "$RPM_BUILD_DIR/RPMS" -name "*.rpm" | head -1) + if [ -n "$RPM_FILE" ]; then + cp "$RPM_FILE" "$ROOT_DIR/" + RPM_NAME=$(basename "$RPM_FILE") + echo "RPM package created at $ROOT_DIR/$RPM_NAME" + fi + fi + fi + + echo "" + echo "RPM Installation Instructions:" + echo " sudo rpm -ivh $RPM_NAME" + echo " # Or to upgrade: sudo rpm -Uvh $RPM_NAME" + echo "" + echo "RPM Removal Instructions:" + echo " sudo rpm -e postgres-dsql" + echo "" +fi + echo "Done!" # For testing, you can: @@ -119,3 +232,7 @@ echo "Done!" # # On Linux, you may also need to ensure the library path is set: # LD_LIBRARY_PATH=/tmp/postgres-dsql/lib /tmp/postgres-dsql/bin/pdsql --version +# +# For RPM testing: +# sudo rpm -ivh postgres-dsql-*.rpm +# pdsql --version From be0473150e894cfb60cf14cb364c156711f377b7 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:26:34 -0700 Subject: [PATCH 13/42] Fix mac build which now requires go --- .github/workflows/build-dsql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index 664d1943dbcd0..24be6193aed01 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -24,7 +24,7 @@ jobs: if: runner.os == 'macOS' run: | brew update - brew install openssl@3 icu4c zip readline + brew install openssl@3 icu4c zip readline go - name: Set up Ubuntu if: runner.os == 'Linux' From 14dc5f7e5a06d0ca2590b814b1a09b424a21a870 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:41:32 -0700 Subject: [PATCH 14/42] dmg --- .github/workflows/build-dsql.yml | 16 ++- .gitignore | 2 + scripts/package.sh | 236 +++++++++++++++++++++++++++++-- 3 files changed, 237 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index 24be6193aed01..1635459766eab 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -62,21 +62,29 @@ jobs: uses: actions/upload-artifact@v4 with: name: postgres-dsql-${{ runner.os }}-${{ runner.arch }} - path: postgres-dsql.zip + path: build/postgres-dsql.zip + + - name: Upload DMG Artifact + if: runner.os == 'macOS' + uses: actions/upload-artifact@v4 + with: + name: postgres-dsql-dmg-${{ runner.arch }} + path: "build/*.dmg" - name: Upload RPM Artifact if: runner.os == 'Linux' uses: actions/upload-artifact@v4 with: name: postgres-dsql-rpm-${{ runner.arch }} - path: "*.rpm" + path: "build/*.rpm" - name: Release if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v1 with: files: | - postgres-dsql.zip - *.rpm + build/postgres-dsql.zip + build/*.dmg + build/*.rpm env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 4e911395fe3ba..85567083dd7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,5 @@ lib*.pc /Release/ /tmp_install/ /portlock/ +# Build artifacts +build/ diff --git a/scripts/package.sh b/scripts/package.sh index 293fc6bc53a6c..5b96a9025bf29 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -10,8 +10,9 @@ echo "================================" # Path setup ROOT_DIR=$(pwd) +BUILD_DIR="$ROOT_DIR/build" DIST_NAME="postgres-dsql" -DIST_DIR="$ROOT_DIR/$DIST_NAME" +DIST_DIR="$BUILD_DIR/$DIST_NAME" SRC_BIN="$ROOT_DIR/src/bin/psql/psql" BINARY_NAME="pdsql" @@ -45,6 +46,9 @@ if [ ! -f "$SRC_LIB" ]; then exit 1 fi +# Create build directory +mkdir -p "$BUILD_DIR" + # Clean any previous packaging attempts if [ -d "$DIST_DIR" ]; then echo "Cleaning previous packaging directory..." @@ -107,17 +111,223 @@ fi # Create a ZIP archive echo "Creating ZIP archive..." ZIP_NAME="${DIST_NAME}.zip" -rm -f "$ZIP_NAME" -(cd "$ROOT_DIR" && zip -r "$ZIP_NAME" "$DIST_NAME") +ZIP_PATH="$BUILD_DIR/$ZIP_NAME" +rm -f "$ZIP_PATH" +(cd "$BUILD_DIR" && zip -r "$ZIP_NAME" "$DIST_NAME") + +echo "Package created at $ZIP_PATH" + +# Create platform-specific installers +if [[ "$PLATFORM" == "macos" ]]; then + echo "Creating DMG package for macOS..." + + # Create a temporary directory for DMG contents + DMG_DIR="$BUILD_DIR/dmg_temp" + rm -rf "$DMG_DIR" + mkdir -p "$DMG_DIR" + + # Create the application bundle structure + APP_NAME="PostgreSQL DSQL.app" + APP_DIR="$DMG_DIR/$APP_NAME" + mkdir -p "$APP_DIR/Contents/MacOS" + mkdir -p "$APP_DIR/Contents/Resources" + + # Copy binaries and libraries to the app bundle + cp -r "$DIST_DIR/bin" "$APP_DIR/Contents/MacOS/" + cp -r "$DIST_DIR/lib" "$APP_DIR/Contents/MacOS/" + + # Create Info.plist for the app bundle + cat > "$APP_DIR/Contents/Info.plist" << 'EOF' + + + + + CFBundleExecutable + pdsql + CFBundleIdentifier + com.aws.postgres-dsql + CFBundleName + PostgreSQL DSQL + CFBundleVersion + 1.0.0 + CFBundleShortVersionString + 1.0.0 + CFBundlePackageType + APPL + LSMinimumSystemVersion + 10.15 + LSApplicationCategoryType + public.app-category.developer-tools + + +EOF + + # Create a simple installer script + INSTALLER_SCRIPT="$DMG_DIR/Install PostgreSQL DSQL.command" + cat > "$INSTALLER_SCRIPT" << 'EOF' +#!/bin/bash +set -e + +echo "Installing PostgreSQL DSQL..." +echo "==============================" + +# Installation directory +INSTALL_DIR="/opt/postgres-dsql" +BIN_LINK="/usr/local/bin/pdsql" + +# Check for admin privileges +if [ "$EUID" -ne 0 ]; then + echo "This installer requires administrator privileges." + echo "Please run with sudo or enter your password when prompted." + exec sudo "$0" "$@" +fi + +# Create installation directory +echo "Creating installation directory at $INSTALL_DIR..." +mkdir -p "$INSTALL_DIR" + +# Get the directory where this script is located (inside the DMG) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +APP_DIR="$SCRIPT_DIR/PostgreSQL DSQL.app" + +# Copy files from the app bundle +echo "Installing PostgreSQL DSQL files..." +cp -r "$APP_DIR/Contents/MacOS/bin" "$INSTALL_DIR/" +cp -r "$APP_DIR/Contents/MacOS/lib" "$INSTALL_DIR/" + +# Create symlink for easy access +echo "Creating symlink at $BIN_LINK..." +mkdir -p "$(dirname "$BIN_LINK")" +ln -sf "$INSTALL_DIR/bin/pdsql" "$BIN_LINK" + +# Set proper permissions +chmod +x "$INSTALL_DIR/bin/pdsql" + +echo "" +echo "Installation completed successfully!" +echo "" +echo "PostgreSQL DSQL has been installed to: $INSTALL_DIR" +echo "You can now use the 'pdsql' command from anywhere in your terminal." +echo "" +echo "Example usage:" +echo " pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "" +echo "To uninstall, run:" +echo " sudo rm -rf $INSTALL_DIR" +echo " sudo rm -f $BIN_LINK" +echo "" + +read -p "Press Enter to close this window..." +EOF + + chmod +x "$INSTALLER_SCRIPT" + + # Create an uninstaller script + UNINSTALLER_SCRIPT="$DMG_DIR/Uninstall PostgreSQL DSQL.command" + cat > "$UNINSTALLER_SCRIPT" << 'EOF' +#!/bin/bash +set -e + +echo "Uninstalling PostgreSQL DSQL..." +echo "================================" -echo "Package created at $ROOT_DIR/$ZIP_NAME" +INSTALL_DIR="/opt/postgres-dsql" +BIN_LINK="/usr/local/bin/pdsql" -# Create RPM package for Linux -if [[ "$PLATFORM" == "linux" ]]; then +# Check for admin privileges +if [ "$EUID" -ne 0 ]; then + echo "This uninstaller requires administrator privileges." + echo "Please run with sudo or enter your password when prompted." + exec sudo "$0" "$@" +fi + +# Remove installation directory +if [ -d "$INSTALL_DIR" ]; then + echo "Removing installation directory..." + rm -rf "$INSTALL_DIR" + echo "Removed $INSTALL_DIR" +else + echo "Installation directory not found: $INSTALL_DIR" +fi + +# Remove symlink +if [ -L "$BIN_LINK" ]; then + echo "Removing symlink..." + rm -f "$BIN_LINK" + echo "Removed $BIN_LINK" +else + echo "Symlink not found: $BIN_LINK" +fi + +echo "" +echo "PostgreSQL DSQL has been completely uninstalled." +echo "" + +read -p "Press Enter to close this window..." +EOF + + chmod +x "$UNINSTALLER_SCRIPT" + + # Create a README file + cat > "$DMG_DIR/README.txt" << 'EOF' +PostgreSQL DSQL Client +====================== + +This package contains the PostgreSQL DSQL client (pdsql) with AWS DSQL authentication support. + +Installation: +1. Double-click "Install PostgreSQL DSQL.command" +2. Enter your administrator password when prompted +3. The pdsql command will be available system-wide + +The client will be installed to /opt/postgres-dsql/ and a symlink will be created at /usr/local/bin/pdsql + +This installation will NOT interfere with any existing PostgreSQL installations on your system. + +Usage: + pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres + +Uninstallation: + Double-click "Uninstall PostgreSQL DSQL.command" + +For more information, visit: https://docs.aws.amazon.com/aurora-dsql/ +EOF + + # Create the DMG + DMG_NAME="postgres-dsql-installer.dmg" + DMG_PATH="$BUILD_DIR/$DMG_NAME" + echo "Creating DMG file..." + + # Remove any existing DMG + rm -f "$DMG_PATH" + + # Create DMG using hdiutil + hdiutil create -volname "PostgreSQL DSQL Installer" \ + -srcfolder "$DMG_DIR" \ + -ov -format UDZO \ + "$DMG_PATH" + + # Clean up temporary directory + rm -rf "$DMG_DIR" + + echo "DMG package created at $DMG_PATH" + echo "" + echo "DMG Installation Instructions:" + echo " 1. Double-click $DMG_NAME to mount" + echo " 2. Double-click 'Install PostgreSQL DSQL.command'" + echo " 3. Enter your administrator password" + echo " 4. Use 'pdsql' command from anywhere" + echo "" + echo "DMG Uninstallation Instructions:" + echo " 1. Mount the DMG again" + echo " 2. Double-click 'Uninstall PostgreSQL DSQL.command'" + echo "" + +elif [[ "$PLATFORM" == "linux" ]]; then echo "Creating RPM package..." # Create RPM build directory structure - RPM_BUILD_DIR="$ROOT_DIR/rpmbuild" + RPM_BUILD_DIR="$BUILD_DIR/rpmbuild" mkdir -p "$RPM_BUILD_DIR"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} # Create spec file @@ -183,9 +393,9 @@ EOF # Find and copy the generated RPM RPM_FILE=$(find "$RPM_BUILD_DIR/RPMS" -name "*.rpm" | head -1) if [ -n "$RPM_FILE" ]; then - cp "$RPM_FILE" "$ROOT_DIR/" + cp "$RPM_FILE" "$BUILD_DIR/" RPM_NAME=$(basename "$RPM_FILE") - echo "RPM package created at $ROOT_DIR/$RPM_NAME" + echo "RPM package created at $BUILD_DIR/$RPM_NAME" else echo "Warning: RPM file not found after build" fi @@ -207,9 +417,9 @@ EOF rpmbuild --define "_topdir $RPM_BUILD_DIR" -bb "$SPEC_FILE" RPM_FILE=$(find "$RPM_BUILD_DIR/RPMS" -name "*.rpm" | head -1) if [ -n "$RPM_FILE" ]; then - cp "$RPM_FILE" "$ROOT_DIR/" + cp "$RPM_FILE" "$BUILD_DIR/" RPM_NAME=$(basename "$RPM_FILE") - echo "RPM package created at $ROOT_DIR/$RPM_NAME" + echo "RPM package created at $BUILD_DIR/$RPM_NAME" fi fi fi @@ -227,12 +437,12 @@ fi echo "Done!" # For testing, you can: -# unzip -o postgres-dsql.zip -d /tmp +# unzip -o build/postgres-dsql.zip -d /tmp # /tmp/postgres-dsql/bin/pdsql --version # # On Linux, you may also need to ensure the library path is set: # LD_LIBRARY_PATH=/tmp/postgres-dsql/lib /tmp/postgres-dsql/bin/pdsql --version # # For RPM testing: -# sudo rpm -ivh postgres-dsql-*.rpm +# sudo rpm -ivh build/postgres-dsql-*.rpm # pdsql --version From d9e538daa68499908c6bbfb5295b768acca65a62 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:43:49 -0700 Subject: [PATCH 15/42] remove dep checks The build process already prints errs --- scripts/build-dsql.sh | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 5a4bd05cb3741..6deb2136aedc4 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -32,44 +32,6 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then # Base configuration for Linux with system packages OS_SPECIFIC_CONFIG="--with-ssl=openssl --with-icu" - - # Check if we have the required development packages - echo " Checking for required development packages..." - - # Check for essential build tools and libraries - MISSING_PACKAGES="" - - if ! dpkg -l | grep -q "libssl-dev\|openssl-dev" 2>/dev/null && ! rpm -qa | grep -q "openssl-devel" 2>/dev/null; then - MISSING_PACKAGES="$MISSING_PACKAGES libssl-dev" - fi - - if ! dpkg -l | grep -q "libreadline-dev\|readline-dev" 2>/dev/null && ! rpm -qa | grep -q "readline-devel" 2>/dev/null; then - MISSING_PACKAGES="$MISSING_PACKAGES libreadline-dev" - fi - - if ! dpkg -l | grep -q "zlib1g-dev\|zlib-dev" 2>/dev/null && ! rpm -qa | grep -q "zlib-devel" 2>/dev/null; then - MISSING_PACKAGES="$MISSING_PACKAGES zlib1g-dev" - fi - - if ! dpkg -l | grep -q "libicu-dev\|icu-dev" 2>/dev/null && ! rpm -qa | grep -q "libicu-devel" 2>/dev/null; then - MISSING_PACKAGES="$MISSING_PACKAGES libicu-dev" - fi - - if ! command -v flex >/dev/null 2>&1; then - MISSING_PACKAGES="$MISSING_PACKAGES flex" - fi - - if ! command -v bison >/dev/null 2>&1; then - MISSING_PACKAGES="$MISSING_PACKAGES bison" - fi - - if [ -n "$MISSING_PACKAGES" ]; then - echo " Warning: Some required packages may be missing: $MISSING_PACKAGES" - echo " On Ubuntu/Debian, install with: sudo apt-get install $MISSING_PACKAGES" - echo " On RHEL/CentOS, install equivalent packages with yum/dnf" - else - echo " Required development packages appear to be installed" - fi else echo "Detected Unix system (assuming Linux-like)" LIBRARY_PATH_VAR="LD_LIBRARY_PATH" From 1141e7754ec98298cbd0d00c04a18bbbf1ba0276 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 13:59:23 -0700 Subject: [PATCH 16/42] Exclude test headers --- src/interfaces/libpq/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 8877713f34c07..a11d4cade264b 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -80,8 +80,8 @@ OBJS += \ win32.o endif -AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include)) -AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a') +AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) +AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') # Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c From 10734cde4f3488ef3b90ee0b40abc8cc3a2707b1 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 14:28:26 -0700 Subject: [PATCH 17/42] More build stuff --- src/interfaces/libpq/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index a11d4cade264b..45c631a73772c 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -83,11 +83,12 @@ endif AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') -# Custom rule for fe-dsql-auth.o that includes AWS libraries +# Custom rule for fe-dsql-auth.o that includes AWS headers (but not libraries) fe-dsql-auth.o: fe-dsql-auth.c - $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o - $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) - rm -f fe-dsql-auth-temp.o + $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o $@ + +# Add AWS libraries to the shared library link +SHLIB_LINK += $(AWS_DSQL_AUTH_LIBS) # Add libraries that libpq depends (or might depend) on into the # shared library link. (The order in which you list them here doesn't From 5680a1567c3dcb47c299f1019921921fd0e69e4f Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 14:50:37 -0700 Subject: [PATCH 18/42] Revert "More build stuff" This reverts commit 10734cde4f3488ef3b90ee0b40abc8cc3a2707b1. --- src/interfaces/libpq/Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 45c631a73772c..a11d4cade264b 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -83,12 +83,11 @@ endif AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') -# Custom rule for fe-dsql-auth.o that includes AWS headers (but not libraries) +# Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c - $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o $@ - -# Add AWS libraries to the shared library link -SHLIB_LINK += $(AWS_DSQL_AUTH_LIBS) + $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o + $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) + rm -f fe-dsql-auth-temp.o # Add libraries that libpq depends (or might depend) on into the # shared library link. (The order in which you list them here doesn't From 2ad1a8c228ae4bc0dca5a5b23ee2a2d6b9d7bd40 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 14:50:59 -0700 Subject: [PATCH 19/42] Inline more --- src/interfaces/libpq/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index a11d4cade264b..4c702acac33a5 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -86,7 +86,7 @@ AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -n # Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o - $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) + $(LD) -r -o $@ fe-dsql-auth-temp.o --whole-archive $(AWS_DSQL_AUTH_LIBS) rm -f fe-dsql-auth-temp.o # Add libraries that libpq depends (or might depend) on into the From 813cd1dcc5008bfce1a12477450d24361c54e556 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 15:00:12 -0700 Subject: [PATCH 20/42] More build and pkg fixes --- scripts/test-packaging.sh | 6 +++--- src/interfaces/libpq/Makefile | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/test-packaging.sh b/scripts/test-packaging.sh index 61ed2d1100a0d..92dd60a0bd126 100755 --- a/scripts/test-packaging.sh +++ b/scripts/test-packaging.sh @@ -29,12 +29,12 @@ fi # Test the packaged binary echo "Testing packaged binary..." -if [ -f "postgres-dsql.zip" ]; then +if [ -f "build/postgres-dsql.zip" ]; then # Clean any previous test rm -rf /tmp/postgres-dsql # Extract and test - unzip -o postgres-dsql.zip -d /tmp + unzip -o build/postgres-dsql.zip -d /tmp echo "Running pdsql --version to verify:" /tmp/postgres-dsql/bin/pdsql --version @@ -46,7 +46,7 @@ if [ -f "postgres-dsql.zip" ]; then exit 1 fi else - echo "❌ Package file not found!" + echo "❌ Package file not found at build/postgres-dsql.zip!" exit 1 fi diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 4c702acac33a5..b3f18e014ab1e 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -86,7 +86,11 @@ AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -n # Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o +ifeq ($(PORTNAME), linux) $(LD) -r -o $@ fe-dsql-auth-temp.o --whole-archive $(AWS_DSQL_AUTH_LIBS) +else + $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) +endif rm -f fe-dsql-auth-temp.o # Add libraries that libpq depends (or might depend) on into the From 639d970acf917d12f80a3f03898b43076d27daa0 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 15:14:10 -0700 Subject: [PATCH 21/42] linux arm builds --- .github/workflows/build-dsql.yml | 6 +++--- scripts/package.sh | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index 1635459766eab..58fd426a13cf0 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - os: [macos-latest, ubuntu-22.04] + os: [macos-latest, ubuntu-22.04, ubuntu-22.04-arm64] runs-on: ${{ matrix.os }} steps: @@ -61,7 +61,7 @@ jobs: - name: Upload Build Artifact uses: actions/upload-artifact@v4 with: - name: postgres-dsql-${{ runner.os }}-${{ runner.arch }} + name: postgres-dsql-${{ matrix.os }}-${{ runner.arch }} path: build/postgres-dsql.zip - name: Upload DMG Artifact @@ -75,7 +75,7 @@ jobs: if: runner.os == 'Linux' uses: actions/upload-artifact@v4 with: - name: postgres-dsql-rpm-${{ runner.arch }} + name: postgres-dsql-rpm-${{ matrix.os }}-${{ runner.arch }} path: "build/*.rpm" - name: Release diff --git a/scripts/package.sh b/scripts/package.sh index 5b96a9025bf29..f65be57d4062c 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -330,16 +330,33 @@ elif [[ "$PLATFORM" == "linux" ]]; then RPM_BUILD_DIR="$BUILD_DIR/rpmbuild" mkdir -p "$RPM_BUILD_DIR"/{BUILD,RPMS,SOURCES,SPECS,SRPMS} + # Detect architecture for RPM + ARCH=$(uname -m) + case "$ARCH" in + x86_64) + RPM_ARCH="x86_64" + ;; + aarch64) + RPM_ARCH="aarch64" + ;; + arm64) + RPM_ARCH="aarch64" + ;; + *) + RPM_ARCH="$ARCH" + ;; + esac + # Create spec file SPEC_FILE="$RPM_BUILD_DIR/SPECS/postgres-dsql.spec" - cat > "$SPEC_FILE" << 'EOF' + cat > "$SPEC_FILE" << EOF Name: postgres-dsql Version: 1.0.0 Release: 1%{?dist} Summary: PostgreSQL DSQL client (pdsql) - AWS DSQL authentication enabled psql License: PostgreSQL URL: https://github.com/your-org/postgres-dsql -BuildArch: x86_64 +BuildArch: $RPM_ARCH %description PostgreSQL DSQL client provides pdsql, a PostgreSQL client with AWS DSQL From 86d9c1556c7e6b70f36305fce9a5432e016e317e Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 15:45:57 -0700 Subject: [PATCH 22/42] Remove the arm64 build it hangs with no logs. should reenable when out of preview --- .github/workflows/build-dsql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index 58fd426a13cf0..87341f0d9d855 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - os: [macos-latest, ubuntu-22.04, ubuntu-22.04-arm64] + os: [macos-latest, ubuntu-22.04] runs-on: ${{ matrix.os }} steps: From ec92c93967818eb9a0ecf2eebd5848a8fde6512a Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:04:54 -0700 Subject: [PATCH 23/42] deb --- .github/workflows/build-dsql.yml | 8 + scripts/install.sh | 303 ++++++++++++++++++++++++------- scripts/package.sh | 82 ++++++++- 3 files changed, 324 insertions(+), 69 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index 87341f0d9d855..e79608fbacba5 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -78,6 +78,13 @@ jobs: name: postgres-dsql-rpm-${{ matrix.os }}-${{ runner.arch }} path: "build/*.rpm" + - name: Upload DEB Artifact + if: runner.os == 'Linux' + uses: actions/upload-artifact@v4 + with: + name: postgres-dsql-deb-${{ matrix.os }}-${{ runner.arch }} + path: "build/*.deb" + - name: Release if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v1 @@ -86,5 +93,6 @@ jobs: build/postgres-dsql.zip build/*.dmg build/*.rpm + build/*.deb env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/install.sh b/scripts/install.sh index cf124cbd87607..318ff8adf30b8 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,81 +10,248 @@ error_exit() { exit 1 } -# Check if running on macOS -if [[ "$OSTYPE" != "darwin"* ]]; then - error_exit "This script is only supported on macOS systems." -fi - -# Check if running on ARM architecture -if [[ "$(uname -m)" != "arm64" ]]; then - error_exit "This script is only intended for ARM-based macOS (Apple Silicon) systems." -fi - -echo "PostgreSQL DSQL ARM macOS Installer" -echo "===================================" - -# Get latest release information using GitHub API -echo "Fetching latest release information..." -RELEASE_INFO=$(curl -s "https://api.github.com/repos/$REPO/releases/latest") -if [[ -z "$RELEASE_INFO" || "$RELEASE_INFO" == *"Not Found"* ]]; then - error_exit "Could not fetch release information. Check your internet connection." -fi - -# Extract download URL for the zip file -DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o '"browser_download_url": *"[^"]*postgres-dsql.zip"' | cut -d'"' -f4) -if [[ -z "$DOWNLOAD_URL" ]]; then - error_exit "No postgres-dsql.zip found in the latest release." -fi - -TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) -echo "Latest release found: $TAG_NAME" - -# Create directories if they don't exist -mkdir -p "$INSTALL_PATH/bin" -mkdir -p "$INSTALL_PATH/lib" - -# Download the release -echo "Downloading release from $DOWNLOAD_URL..." -TEMP_DIR=$(mktemp -d) -curl -L "$DOWNLOAD_URL" -o "$TEMP_DIR/postgres-dsql.zip" - -# Extract the release -echo "Extracting files to $INSTALL_PATH..." -unzip -o "$TEMP_DIR/postgres-dsql.zip" -d "$TEMP_DIR" +# Function to detect OS +detect_os() { + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "macos" + elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "linux" + else + error_exit "Unsupported operating system: $OSTYPE" + fi +} -# Copy files to install location -cp -r "$TEMP_DIR/postgres-dsql/bin/"* "$INSTALL_PATH/bin/" -cp -r "$TEMP_DIR/postgres-dsql/lib/"* "$INSTALL_PATH/lib/" +# Function to detect architecture +detect_arch() { + local arch=$(uname -m) + case "$arch" in + x86_64) + echo "X64" + ;; + aarch64|arm64) + echo "ARM64" + ;; + *) + error_exit "Unsupported architecture: $arch" + ;; + esac +} -# Clean up temp files -rm -rf "$TEMP_DIR" +# Function to detect Linux distribution +detect_linux_distro() { + if command -v apt-get >/dev/null 2>&1; then + echo "debian" + elif command -v yum >/dev/null 2>&1 || command -v dnf >/dev/null 2>&1; then + echo "rhel" + else + echo "unknown" + fi +} -# Make the binary executable -chmod +x "$INSTALL_PATH/bin/pdsql" +# Function to install via package manager (Linux) +install_via_package() { + local os="$1" + local arch="$2" + local distro="$3" + + echo "Attempting package manager installation..." + + # Get latest release information + echo "Fetching latest release information..." + RELEASE_INFO=$(curl -s "https://api.github.com/repos/$REPO/releases/latest") + if [[ -z "$RELEASE_INFO" || "$RELEASE_INFO" == *"Not Found"* ]]; then + error_exit "Could not fetch release information. Check your internet connection." + fi + + TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) + echo "Latest release found: $TAG_NAME" + + # Determine package type and download URL + local package_type="" + local download_url="" + local package_file="" + + if [[ "$distro" == "debian" ]]; then + package_type="deb" + # Convert arch format for DEB (X64 -> amd64, ARM64 -> arm64) + local deb_arch="" + if [[ "$arch" == "X64" ]]; then + deb_arch="amd64" + elif [[ "$arch" == "ARM64" ]]; then + deb_arch="arm64" + fi + package_file="postgres-dsql_1.0.0-1_${deb_arch}.deb" + elif [[ "$distro" == "rhel" ]]; then + package_type="rpm" + # Convert arch format for RPM (X64 -> x86_64, ARM64 -> aarch64) + local rpm_arch="" + if [[ "$arch" == "X64" ]]; then + rpm_arch="x86_64" + elif [[ "$arch" == "ARM64" ]]; then + rpm_arch="aarch64" + fi + package_file="postgres-dsql-1.0.0-1.${rpm_arch}.rpm" + else + echo "Unknown Linux distribution, falling back to ZIP installation..." + return 1 + fi + + # Find download URL for the package + download_url=$(echo "$RELEASE_INFO" | grep -o "\"browser_download_url\": *\"[^\"]*${package_file}\"" | cut -d'"' -f4) + if [[ -z "$download_url" ]]; then + echo "Package ${package_file} not found in release, falling back to ZIP installation..." + return 1 + fi + + # Download and install package + echo "Downloading ${package_type} package..." + TEMP_DIR=$(mktemp -d) + local temp_package="$TEMP_DIR/$package_file" + curl -L "$download_url" -o "$temp_package" + + echo "Installing package (may require sudo password)..." + if [[ "$package_type" == "deb" ]]; then + if command -v apt >/dev/null 2>&1; then + sudo apt install -y "$temp_package" + else + sudo dpkg -i "$temp_package" + # Fix dependencies if needed + sudo apt-get install -f -y 2>/dev/null || true + fi + elif [[ "$package_type" == "rpm" ]]; then + if command -v dnf >/dev/null 2>&1; then + sudo dnf install -y "$temp_package" + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y "$temp_package" + else + sudo rpm -ivh "$temp_package" + fi + fi + + # Clean up + rm -rf "$TEMP_DIR" + + echo "Package installation completed successfully!" + echo "PostgreSQL DSQL (pdsql) is now available system-wide." + return 0 +} -echo "Installation completed successfully!" -echo "PostgreSQL DSQL (pdsql) installed to: $INSTALL_PATH/bin/pdsql" +# Function to install via ZIP extraction +install_via_zip() { + local os="$1" + local arch="$2" + + echo "Installing via ZIP extraction..." + + # Get latest release information + echo "Fetching latest release information..." + RELEASE_INFO=$(curl -s "https://api.github.com/repos/$REPO/releases/latest") + if [[ -z "$RELEASE_INFO" || "$RELEASE_INFO" == *"Not Found"* ]]; then + error_exit "Could not fetch release information. Check your internet connection." + fi + + TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) + echo "Latest release found: $TAG_NAME" + + # Find appropriate ZIP file + local zip_pattern="" + if [[ "$os" == "macos" ]]; then + zip_pattern="postgres-dsql-macos-latest-${arch}" + elif [[ "$os" == "linux" ]]; then + zip_pattern="postgres-dsql-ubuntu-22.04-${arch}" + fi + + # Extract download URL for the zip file + DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o "\"browser_download_url\": *\"[^\"]*${zip_pattern}[^\"]*\.zip\"" | cut -d'"' -f4) + if [[ -z "$DOWNLOAD_URL" ]]; then + # Fallback to generic postgres-dsql.zip + DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o '"browser_download_url": *"[^"]*postgres-dsql.zip"' | cut -d'"' -f4) + if [[ -z "$DOWNLOAD_URL" ]]; then + error_exit "No compatible ZIP file found in the latest release." + fi + fi + + # Create directories if they don't exist + mkdir -p "$INSTALL_PATH/bin" + mkdir -p "$INSTALL_PATH/lib" + + # Download the release + echo "Downloading release from $DOWNLOAD_URL..." + TEMP_DIR=$(mktemp -d) + curl -L "$DOWNLOAD_URL" -o "$TEMP_DIR/postgres-dsql.zip" + + # Extract the release + echo "Extracting files to $INSTALL_PATH..." + unzip -o "$TEMP_DIR/postgres-dsql.zip" -d "$TEMP_DIR" + + # Copy files to install location + cp -r "$TEMP_DIR/postgres-dsql/bin/"* "$INSTALL_PATH/bin/" + cp -r "$TEMP_DIR/postgres-dsql/lib/"* "$INSTALL_PATH/lib/" + + # Clean up temp files + rm -rf "$TEMP_DIR" + + # Make the binary executable + chmod +x "$INSTALL_PATH/bin/pdsql" + + echo "ZIP installation completed successfully!" + echo "PostgreSQL DSQL (pdsql) installed to: $INSTALL_PATH/bin/pdsql" + + # Check if installation path is in PATH + if [[ ":$PATH:" != *":$INSTALL_PATH/bin:"* ]]; then + echo "" + echo "NOTICE: Your PATH environment variable doesn't contain $INSTALL_PATH/bin" + echo "To add it to your PATH, add the following line to your shell configuration file:" + echo "" + echo " export PATH=\"$INSTALL_PATH/bin:\$PATH\"" + echo "" + echo "Shell configuration files:" + echo " - Bash: $HOME/.bashrc or $HOME/.bash_profile" + echo " - Zsh: $HOME/.zshrc" + echo " - Fish: $HOME/.config/fish/config.fish" + echo "" + echo "Then, reload your shell configuration or restart your terminal." + fi +} -# Check if installation path is in PATH -if [[ ":$PATH:" != *":$INSTALL_PATH/bin:"* ]]; then +# Main installation logic +main() { + echo "PostgreSQL DSQL Universal Installer" + echo "===================================" + + # Detect system information + OS=$(detect_os) + ARCH=$(detect_arch) + + echo "Detected system: $OS $ARCH" + + if [[ "$OS" == "linux" ]]; then + DISTRO=$(detect_linux_distro) + echo "Detected Linux distribution type: $DISTRO" + + # Try package manager installation first, fall back to ZIP if it fails + if ! install_via_package "$OS" "$ARCH" "$DISTRO"; then + echo "Package installation failed or unavailable, trying ZIP installation..." + install_via_zip "$OS" "$ARCH" + fi + else + # macOS - use ZIP installation + install_via_zip "$OS" "$ARCH" + fi + echo "" - echo "NOTICE: Your PATH environment variable doesn't contain $INSTALL_PATH/bin" - echo "To add it to your PATH, add the following line to your $HOME/.bashrc or $HOME/.zshrc file:" + echo "Installation completed! To verify, run:" echo "" - echo " export PATH=\"$INSTALL_PATH/bin:\$PATH\"" + echo " pdsql --version" echo "" - echo "Then, reload your shell configuration by running:" + echo "For usage help, run:" echo "" - if [[ "$SHELL" == *"zsh"* ]]; then - echo " source $HOME/.zshrc" - else - echo " source $HOME/.bashrc" - fi -fi + echo " pdsql --help" + echo "" + echo "Example DSQL connection:" + echo "" + echo " pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +} -echo "" -echo "To verify installation, run:" -echo "" -echo " pdsql --version" -echo "" -echo "If the command is not found, ensure your PATH is set correctly as described above." +# Run main function +main "$@" diff --git a/scripts/package.sh b/scripts/package.sh index f65be57d4062c..d8e969e874946 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -389,7 +389,7 @@ ln -s /opt/postgres-dsql/bin/pdsql %{buildroot}/usr/bin/pdsql %post echo "PostgreSQL DSQL client installed successfully!" echo "Use 'pdsql' command to connect to AWS DSQL databases." -echo "Example: pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "Example: pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" %changelog * $(date +'%a %b %d %Y') Build System - 1.0.0-1 @@ -449,6 +449,86 @@ EOF echo "RPM Removal Instructions:" echo " sudo rpm -e postgres-dsql" echo "" + + # Create DEB package + echo "Creating DEB package..." + + # Detect architecture for DEB + case "$ARCH" in + x86_64) + DEB_ARCH="amd64" + ;; + aarch64) + DEB_ARCH="arm64" + ;; + arm64) + DEB_ARCH="arm64" + ;; + *) + DEB_ARCH="$ARCH" + ;; + esac + + # Create DEB build directory structure + DEB_BUILD_DIR="$BUILD_DIR/debbuild" + DEB_PKG_DIR="$DEB_BUILD_DIR/postgres-dsql_1.0.0-1_$DEB_ARCH" + mkdir -p "$DEB_PKG_DIR"/{DEBIAN,opt/postgres-dsql/{bin,lib},usr/bin} + + # Copy files + cp "$DIST_DIR/bin/pdsql" "$DEB_PKG_DIR/opt/postgres-dsql/bin/" + cp "$DIST_DIR/lib"/* "$DEB_PKG_DIR/opt/postgres-dsql/lib/" + + # Create symlink + ln -s /opt/postgres-dsql/bin/pdsql "$DEB_PKG_DIR/usr/bin/pdsql" + + # Create control file + cat > "$DEB_PKG_DIR/DEBIAN/control" << EOF +Package: postgres-dsql +Version: 1.0.0-1 +Section: database +Priority: optional +Architecture: $DEB_ARCH +Maintainer: Build System +Description: PostgreSQL DSQL client (pdsql) - AWS DSQL authentication enabled psql + PostgreSQL DSQL client provides pdsql, a PostgreSQL client with AWS DSQL + authentication support. This package installs alongside existing PostgreSQL + installations without conflicts by using different binary and library names. +Homepage: https://github.com/your-org/postgres-dsql +EOF + + # Create postinst script + cat > "$DEB_PKG_DIR/DEBIAN/postinst" << 'EOF' +#!/bin/bash +echo "PostgreSQL DSQL client installed successfully!" +echo "Use 'pdsql' command to connect to AWS DSQL databases." +echo "Example: pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +EOF + chmod 755 "$DEB_PKG_DIR/DEBIAN/postinst" + + # Build the DEB package + echo "Building DEB package..." + if command -v dpkg-deb >/dev/null 2>&1; then + dpkg-deb --build "$DEB_PKG_DIR" + DEB_FILE="$DEB_PKG_DIR.deb" + if [ -f "$DEB_FILE" ]; then + mv "$DEB_FILE" "$BUILD_DIR/" + DEB_NAME=$(basename "$DEB_FILE") + echo "DEB package created at $BUILD_DIR/$DEB_NAME" + else + echo "Warning: DEB file not found after build" + fi + else + echo "Warning: dpkg-deb not available. DEB package not created." + fi + + echo "" + echo "DEB Installation Instructions:" + echo " sudo dpkg -i $DEB_NAME" + echo " # Or: sudo apt install ./$DEB_NAME" + echo "" + echo "DEB Removal Instructions:" + echo " sudo apt remove postgres-dsql" + echo "" fi echo "Done!" From d159189e76787ea7bd45284dda06faa90937a166 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:17:43 -0700 Subject: [PATCH 24/42] Fix releases --- .github/workflows/build-dsql.yml | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index e79608fbacba5..ba6697048bd04 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -3,9 +3,10 @@ name: Build and Package DSQL on: workflow_dispatch: # Allow manual triggering push: - branches: ["*"] # Run on all branch pushes + branches: ["*"] # Run on all branch pushes (CI) + tags: ['v*'] # Run on version tags (Release) pull_request: - branches: ["*"] # Run on all pull requests + branches: ["*"] # Run on all pull requests (CI) jobs: build: @@ -85,14 +86,27 @@ jobs: name: postgres-dsql-deb-${{ matrix.os }}-${{ runner.arch }} path: "build/*.deb" - - name: Release - if: startsWith(github.ref, 'refs/tags/') + release: + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Display structure of downloaded files + run: ls -R artifacts + + - name: Create Release uses: softprops/action-gh-release@v1 with: files: | - build/postgres-dsql.zip - build/*.dmg - build/*.rpm - build/*.deb + artifacts/**/*.zip + artifacts/**/*.dmg + artifacts/**/*.rpm + artifacts/**/*.deb + generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From aa18d0ed938773569b8f60c9bdc697e153b801df Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 20 Jun 2025 16:36:57 -0700 Subject: [PATCH 25/42] Fix dup zips --- .github/workflows/build-dsql.yml | 38 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index ba6697048bd04..d76cddc5f129f 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -99,14 +99,42 @@ jobs: - name: Display structure of downloaded files run: ls -R artifacts + - name: Prepare release files + run: | + mkdir -p release-files + + # Rename ZIP files to be unique + find artifacts -name "postgres-dsql.zip" | while read file; do + dir=$(dirname "$file") + artifact_name=$(basename "$dir") + # Extract platform info from artifact name + if [[ "$artifact_name" == *"macos"* ]]; then + if [[ "$artifact_name" == *"ARM64"* ]]; then + cp "$file" "release-files/postgres-dsql-macos-arm64.zip" + else + cp "$file" "release-files/postgres-dsql-macos-x64.zip" + fi + elif [[ "$artifact_name" == *"ubuntu"* ]]; then + if [[ "$artifact_name" == *"ARM64"* ]]; then + cp "$file" "release-files/postgres-dsql-linux-arm64.zip" + else + cp "$file" "release-files/postgres-dsql-linux-x64.zip" + fi + fi + done + + # Copy other files with original names + find artifacts -name "*.dmg" -exec cp {} release-files/ \; + find artifacts -name "*.rpm" -exec cp {} release-files/ \; + find artifacts -name "*.deb" -exec cp {} release-files/ \; + + echo "Release files prepared:" + ls -la release-files/ + - name: Create Release uses: softprops/action-gh-release@v1 with: - files: | - artifacts/**/*.zip - artifacts/**/*.dmg - artifacts/**/*.rpm - artifacts/**/*.deb + files: release-files/* generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7a713f7e5572faeaf00949075eee454e8486d969 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 20:59:13 -0700 Subject: [PATCH 26/42] Remove dmg --- .github/workflows/build-dsql.yml | 7 -- scripts/install.sh | 27 ++-- scripts/package.sh | 209 +------------------------------ 3 files changed, 18 insertions(+), 225 deletions(-) diff --git a/.github/workflows/build-dsql.yml b/.github/workflows/build-dsql.yml index d76cddc5f129f..d51943511bf0e 100644 --- a/.github/workflows/build-dsql.yml +++ b/.github/workflows/build-dsql.yml @@ -65,12 +65,6 @@ jobs: name: postgres-dsql-${{ matrix.os }}-${{ runner.arch }} path: build/postgres-dsql.zip - - name: Upload DMG Artifact - if: runner.os == 'macOS' - uses: actions/upload-artifact@v4 - with: - name: postgres-dsql-dmg-${{ runner.arch }} - path: "build/*.dmg" - name: Upload RPM Artifact if: runner.os == 'Linux' @@ -124,7 +118,6 @@ jobs: done # Copy other files with original names - find artifacts -name "*.dmg" -exec cp {} release-files/ \; find artifacts -name "*.rpm" -exec cp {} release-files/ \; find artifacts -name "*.deb" -exec cp {} release-files/ \; diff --git a/scripts/install.sh b/scripts/install.sh index 318ff8adf30b8..7959634c7d33b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -153,22 +153,27 @@ install_via_zip() { TAG_NAME=$(echo "$RELEASE_INFO" | grep -o '"tag_name": *"[^"]*"' | cut -d'"' -f4) echo "Latest release found: $TAG_NAME" - # Find appropriate ZIP file - local zip_pattern="" + # Find appropriate ZIP file based on OS and architecture + local zip_filename="" + local arch_suffix="" + + # Convert architecture format + if [[ "$arch" == "X64" ]]; then + arch_suffix="x64" + elif [[ "$arch" == "ARM64" ]]; then + arch_suffix="arm64" + fi + if [[ "$os" == "macos" ]]; then - zip_pattern="postgres-dsql-macos-latest-${arch}" + zip_filename="postgres-dsql-macos-${arch_suffix}.zip" elif [[ "$os" == "linux" ]]; then - zip_pattern="postgres-dsql-ubuntu-22.04-${arch}" + zip_filename="postgres-dsql-linux-${arch_suffix}.zip" fi - # Extract download URL for the zip file - DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o "\"browser_download_url\": *\"[^\"]*${zip_pattern}[^\"]*\.zip\"" | cut -d'"' -f4) + # Find download URL for the platform-specific zip file + DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o "\"browser_download_url\": *\"[^\"]*${zip_filename}\"" | cut -d'"' -f4) if [[ -z "$DOWNLOAD_URL" ]]; then - # Fallback to generic postgres-dsql.zip - DOWNLOAD_URL=$(echo "$RELEASE_INFO" | grep -o '"browser_download_url": *"[^"]*postgres-dsql.zip"' | cut -d'"' -f4) - if [[ -z "$DOWNLOAD_URL" ]]; then - error_exit "No compatible ZIP file found in the latest release." - fi + error_exit "${zip_filename} not found in the latest release." fi # Create directories if they don't exist diff --git a/scripts/package.sh b/scripts/package.sh index d8e969e874946..851dd8ebf5c9a 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -117,213 +117,8 @@ rm -f "$ZIP_PATH" echo "Package created at $ZIP_PATH" -# Create platform-specific installers -if [[ "$PLATFORM" == "macos" ]]; then - echo "Creating DMG package for macOS..." - - # Create a temporary directory for DMG contents - DMG_DIR="$BUILD_DIR/dmg_temp" - rm -rf "$DMG_DIR" - mkdir -p "$DMG_DIR" - - # Create the application bundle structure - APP_NAME="PostgreSQL DSQL.app" - APP_DIR="$DMG_DIR/$APP_NAME" - mkdir -p "$APP_DIR/Contents/MacOS" - mkdir -p "$APP_DIR/Contents/Resources" - - # Copy binaries and libraries to the app bundle - cp -r "$DIST_DIR/bin" "$APP_DIR/Contents/MacOS/" - cp -r "$DIST_DIR/lib" "$APP_DIR/Contents/MacOS/" - - # Create Info.plist for the app bundle - cat > "$APP_DIR/Contents/Info.plist" << 'EOF' - - - - - CFBundleExecutable - pdsql - CFBundleIdentifier - com.aws.postgres-dsql - CFBundleName - PostgreSQL DSQL - CFBundleVersion - 1.0.0 - CFBundleShortVersionString - 1.0.0 - CFBundlePackageType - APPL - LSMinimumSystemVersion - 10.15 - LSApplicationCategoryType - public.app-category.developer-tools - - -EOF - - # Create a simple installer script - INSTALLER_SCRIPT="$DMG_DIR/Install PostgreSQL DSQL.command" - cat > "$INSTALLER_SCRIPT" << 'EOF' -#!/bin/bash -set -e - -echo "Installing PostgreSQL DSQL..." -echo "==============================" - -# Installation directory -INSTALL_DIR="/opt/postgres-dsql" -BIN_LINK="/usr/local/bin/pdsql" - -# Check for admin privileges -if [ "$EUID" -ne 0 ]; then - echo "This installer requires administrator privileges." - echo "Please run with sudo or enter your password when prompted." - exec sudo "$0" "$@" -fi - -# Create installation directory -echo "Creating installation directory at $INSTALL_DIR..." -mkdir -p "$INSTALL_DIR" - -# Get the directory where this script is located (inside the DMG) -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -APP_DIR="$SCRIPT_DIR/PostgreSQL DSQL.app" - -# Copy files from the app bundle -echo "Installing PostgreSQL DSQL files..." -cp -r "$APP_DIR/Contents/MacOS/bin" "$INSTALL_DIR/" -cp -r "$APP_DIR/Contents/MacOS/lib" "$INSTALL_DIR/" - -# Create symlink for easy access -echo "Creating symlink at $BIN_LINK..." -mkdir -p "$(dirname "$BIN_LINK")" -ln -sf "$INSTALL_DIR/bin/pdsql" "$BIN_LINK" - -# Set proper permissions -chmod +x "$INSTALL_DIR/bin/pdsql" - -echo "" -echo "Installation completed successfully!" -echo "" -echo "PostgreSQL DSQL has been installed to: $INSTALL_DIR" -echo "You can now use the 'pdsql' command from anywhere in your terminal." -echo "" -echo "Example usage:" -echo " pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" -echo "" -echo "To uninstall, run:" -echo " sudo rm -rf $INSTALL_DIR" -echo " sudo rm -f $BIN_LINK" -echo "" - -read -p "Press Enter to close this window..." -EOF - - chmod +x "$INSTALLER_SCRIPT" - - # Create an uninstaller script - UNINSTALLER_SCRIPT="$DMG_DIR/Uninstall PostgreSQL DSQL.command" - cat > "$UNINSTALLER_SCRIPT" << 'EOF' -#!/bin/bash -set -e - -echo "Uninstalling PostgreSQL DSQL..." -echo "================================" - -INSTALL_DIR="/opt/postgres-dsql" -BIN_LINK="/usr/local/bin/pdsql" - -# Check for admin privileges -if [ "$EUID" -ne 0 ]; then - echo "This uninstaller requires administrator privileges." - echo "Please run with sudo or enter your password when prompted." - exec sudo "$0" "$@" -fi - -# Remove installation directory -if [ -d "$INSTALL_DIR" ]; then - echo "Removing installation directory..." - rm -rf "$INSTALL_DIR" - echo "Removed $INSTALL_DIR" -else - echo "Installation directory not found: $INSTALL_DIR" -fi - -# Remove symlink -if [ -L "$BIN_LINK" ]; then - echo "Removing symlink..." - rm -f "$BIN_LINK" - echo "Removed $BIN_LINK" -else - echo "Symlink not found: $BIN_LINK" -fi - -echo "" -echo "PostgreSQL DSQL has been completely uninstalled." -echo "" - -read -p "Press Enter to close this window..." -EOF - - chmod +x "$UNINSTALLER_SCRIPT" - - # Create a README file - cat > "$DMG_DIR/README.txt" << 'EOF' -PostgreSQL DSQL Client -====================== - -This package contains the PostgreSQL DSQL client (pdsql) with AWS DSQL authentication support. - -Installation: -1. Double-click "Install PostgreSQL DSQL.command" -2. Enter your administrator password when prompted -3. The pdsql command will be available system-wide - -The client will be installed to /opt/postgres-dsql/ and a symlink will be created at /usr/local/bin/pdsql - -This installation will NOT interfere with any existing PostgreSQL installations on your system. - -Usage: - pdsql --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres - -Uninstallation: - Double-click "Uninstall PostgreSQL DSQL.command" - -For more information, visit: https://docs.aws.amazon.com/aurora-dsql/ -EOF - - # Create the DMG - DMG_NAME="postgres-dsql-installer.dmg" - DMG_PATH="$BUILD_DIR/$DMG_NAME" - echo "Creating DMG file..." - - # Remove any existing DMG - rm -f "$DMG_PATH" - - # Create DMG using hdiutil - hdiutil create -volname "PostgreSQL DSQL Installer" \ - -srcfolder "$DMG_DIR" \ - -ov -format UDZO \ - "$DMG_PATH" - - # Clean up temporary directory - rm -rf "$DMG_DIR" - - echo "DMG package created at $DMG_PATH" - echo "" - echo "DMG Installation Instructions:" - echo " 1. Double-click $DMG_NAME to mount" - echo " 2. Double-click 'Install PostgreSQL DSQL.command'" - echo " 3. Enter your administrator password" - echo " 4. Use 'pdsql' command from anywhere" - echo "" - echo "DMG Uninstallation Instructions:" - echo " 1. Mount the DMG again" - echo " 2. Double-click 'Uninstall PostgreSQL DSQL.command'" - echo "" - -elif [[ "$PLATFORM" == "linux" ]]; then +# Create platform-specific packages for Linux only +if [[ "$PLATFORM" == "linux" ]]; then echo "Creating RPM package..." # Create RPM build directory structure From b261241acc340f3f758ddf71ab2e6bdb0162ebe2 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:05:28 -0700 Subject: [PATCH 27/42] Update readme for dsql --- README_DSQL.md | 367 ++++++++++++++++--------------------------------- 1 file changed, 122 insertions(+), 245 deletions(-) diff --git a/README_DSQL.md b/README_DSQL.md index 2d315d363eb40..83cc1091f468c 100644 --- a/README_DSQL.md +++ b/README_DSQL.md @@ -1,328 +1,205 @@ -# PostgreSQL Command Line Client (psql) with AWS DSQL Authentication +# PostgreSQL DSQL Client (pdsql) -This document describes how to build the PostgreSQL command line client (`psql`) with AWS DSQL Authentication support, which allows automatic token generation when connecting to AWS Database Services for PostgreSQL. +A PostgreSQL command-line client with built-in AWS DSQL authentication support. Connect to AWS DSQL databases with automatic token generation - no manual token management required. -## Overview of DSQL Authentication +## 🚀 Quick Installation -DSQL Authentication allows `psql` to automatically generate temporary authentication tokens when connecting to AWS Database Services for PostgreSQL. This integration offers several advantages: - -- No need to manually generate and paste tokens -- Automatic token regeneration if connections fail or tokens expire -- Simplified connection experience with a single `--dsql` flag - -## Prerequisites - -- C compiler (GCC or Clang) -- Make -- Readline library (for command line editing) -- OpenSSL development libraries (required for SSL support) -- AWS credentials configured in your environment (via ~/.aws/credentials, environment variables, etc.) -- AWS DSQL Auth library (located in `aws-dsql-auth` directory) - -## Build Steps - -### Quick Build with build-dsql.sh Script - -The easiest way to build PostgreSQL with DSQL Authentication support is to use the provided `build-dsql.sh` script: +Install `pdsql` with a single command: ```bash -# Make the script executable if needed -chmod +x build-dsql.sh - -# Run the build script -./build-dsql.sh +curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/main/scripts/install.sh | sh ``` -This script automates the following tasks: -1. Initializes the aws-dsql-auth submodules if needed -2. Builds the AWS DSQL Auth library -3. Configures PostgreSQL with SSL support -4. Builds libpq and psql -5. Verifies SSL support -6. Displays usage instructions +This installer automatically detects your platform (macOS/Linux) and architecture, downloads the appropriate package, and installs `pdsql` to your local environment. -The script automatically detects your OS and sets appropriate configuration options. After running the script, you'll have a working psql binary with DSQL Authentication support. +### Manual Installation -### Manual Build Steps +If you prefer to download manually: -If you prefer to build the components step-by-step, follow these instructions: +1. Visit the [GitHub Releases page](https://github.com/marcbowes/postgres/releases) +2. Download the appropriate package for your platform: + - **macOS Intel**: `postgres-dsql-macos-x64.zip` + - **macOS Apple Silicon**: `postgres-dsql-macos-arm64.zip` + - **Linux x64**: `postgres-dsql-linux-x64.zip` + - **Linux ARM64**: `postgres-dsql-linux-arm64.zip` +3. Extract and run: + ```bash + unzip postgres-dsql-*.zip + cd postgres-dsql + ./bin/pdsql --version + ``` -#### 1. Install Required Dependencies +### Package Manager Installation (Linux) -##### On macOS: -```bash -brew install openssl readline -``` +For Linux users, we also provide native packages: -##### On Ubuntu/Debian: +**Debian/Ubuntu (.deb)**: ```bash -sudo apt-get install libssl-dev libreadline-dev +wget https://github.com/marcbowes/postgres/releases/latest/download/postgres-dsql_1.0.0-1_amd64.deb +sudo apt install ./postgres-dsql_1.0.0-1_amd64.deb ``` -##### On RHEL/Fedora: +**RHEL/Fedora (.rpm)**: ```bash -sudo dnf install openssl-devel readline-devel +wget https://github.com/marcbowes/postgres/releases/latest/download/postgres-dsql-1.0.0-1.x86_64.rpm +sudo dnf install postgres-dsql-1.0.0-1.x86_64.rpm ``` -#### 2. Build the AWS DSQL Auth Library - -Initialize submodules (if not already done) and build the AWS DSQL Auth library: - -```bash -cd aws-dsql-auth -git submodule update --init --recursive -./build.sh -cd .. -``` +## 🔧 Usage -#### 3. Configure PostgreSQL with SSL Support +### Basic Connection -Configure the PostgreSQL source code with SSL support enabled: +Connect to an AWS DSQL database: ```bash -# macOS (with Homebrew) -./configure --with-openssl --with-includes=/opt/homebrew/opt/openssl/include --with-libraries=/opt/homebrew/opt/openssl/lib - -# Linux (typically SSL is found automatically) -./configure --with-openssl +pdsql --host=your-dsql-endpoint.example.com --user=admin --port=5432 --dbname=postgres ``` -Additional common options include: -- `--prefix=/path/to/install` - Set installation directory -- `--without-icu` - Disable ICU support if not available -- `--enable-tap-tests` - Enable TAP tests if you plan to run test suites +### Connection String Format -#### 4. Build libpq (PostgreSQL Client Library) - -Build the PostgreSQL client library that psql depends on: +You can also use PostgreSQL connection strings: ```bash -make -C src/interfaces/libpq +pdsql "host=your-dsql-endpoint.example.com user=admin port=5432 dbname=postgres" ``` -#### 5. Build psql +### Key Features -Build the psql command line utility: +- **Automatic Authentication**: No need to manually generate or manage tokens +- **Secure by Default**: Automatically enforces SSL connections +- **Token Auto-Renewal**: Handles token expiration transparently +- **Standard psql Interface**: All familiar psql commands and features work -```bash -make -C src/bin/psql -``` +### How It Works -#### 6. Verify SSL Support +When you connect with `pdsql`: -Ensure SSL support was built properly: +1. **SSL Required**: Automatically enforces secure connections +2. **Token Generation**: Generates temporary AWS authentication tokens automatically +3. **Admin Privileges**: When connecting as `admin` user, full admin privileges are granted +4. **Auto-Renewal**: New tokens are generated for each connection attempt +5. **Short-Lived Tokens**: Tokens expire after 5 seconds for enhanced security -```bash -src/bin/psql/psql --help | grep -i ssl -``` -You should see SSL-related options, or alternatively: -```bash -ldd src/bin/psql/psql | grep -i ssl # Linux -otool -L src/bin/psql/psql | grep -i ssl # macOS -``` +## 🔐 AWS Credentials Setup -### 7. Running psql - -To run the newly built psql with the locally built libpq: +`pdsql` uses your existing AWS credentials. Ensure you have credentials configured through one of these methods: +### AWS CLI (Recommended) ```bash -# On macOS -DYLD_LIBRARY_PATH=$(pwd)/src/interfaces/libpq ./src/bin/psql/psql [options] - -# On Linux -LD_LIBRARY_PATH=$(pwd)/src/interfaces/libpq ./src/bin/psql/psql [options] +aws configure ``` -Note: The AWS DSQL Auth library is statically linked into psql, so you only need to include the libpq library path. - -## Using DSQL Authentication - -To connect using DSQL authentication, add the `--dsql` flag to your `psql` command: - +### Environment Variables ```bash -./src/bin/psql/psql --dsql --host=your-dsql-endpoint.example.com --user=admin +export AWS_ACCESS_KEY_ID=your_access_key +export AWS_SECRET_ACCESS_KEY=your_secret_key +export AWS_REGION=us-east-1 # Optional ``` -Or, with the full connection string: - -```bash -./src/bin/psql/psql --dsql "dbname=postgres user=admin host=your-dsql-endpoint.example.com" +### AWS Credentials File +Create `~/.aws/credentials`: +```ini +[default] +aws_access_key_id = your_access_key +aws_secret_access_key = your_secret_key ``` -### How DSQL Authentication Works +### IAM Roles (EC2/ECS/Lambda) +If running on AWS infrastructure, `pdsql` will automatically use IAM roles. -When the `--dsql` flag is provided: - -1. `psql` automatically sets `SSLMODE=require` to ensure a secure connection (SSL must be compiled in) -2. When connecting to a server, DSQL token generation is triggered -3. If the connection user is `admin`, the token is generated with admin privileges -4. A new token is generated for each connection attempt to ensure security -5. Tokens are set to expire after 5 seconds to enhance security -6. If the connection fails and reconnection is attempted, a new token is generated automatically - -### Environment Variables - -- `PGSSLMODE`: If not already set and `--dsql` is used, it will be set to `require` -- `AWS_REGION`: Optional, if not set the region will be inferred from the hostname -- `PGSSLROOTCERT`: Optional, path to SSL root certificate for verification - -## Testing - -To run tests specifically for libpq and psql: +## 📋 Examples +### Interactive Session ```bash -# Test libpq -make -C src/interfaces/libpq check +# Start an interactive session +pdsql --host=workgroup.123456789012.us-east-1.dsql.amazonaws.com --user=admin -# Test psql -make -C src/bin/psql check +# Once connected, you can run SQL commands: +postgres=> \l +postgres=> CREATE TABLE users (id serial, name text); +postgres=> INSERT INTO users (name) VALUES ('Alice'); +postgres=> SELECT * FROM users; ``` -Note: For comprehensive testing, configure with `--enable-tap-tests` before running these commands. - -## Optional: Installation +### One-liner Queries +```bash +# Execute a single query +pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin -c "SELECT version();" -If you want to install psql and libpq: +# Execute SQL from a file +pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin -f queries.sql +``` +### Connection with Options ```bash -make -C src/bin/psql install -make -C src/interfaces/libpq install -make -C src/include install # For header files +# Connect with specific database and additional options +pdsql --host=your-endpoint.dsql.amazonaws.com \ + --user=admin \ + --port=5432 \ + --dbname=postgres \ + --echo-queries \ + --no-password ``` -## Troubleshooting - -### SSL-Related Issues - -1. If you see "sslmode value 'require' invalid when SSL support is not compiled in": - - Ensure you configured with `--with-openssl` - - Check that OpenSSL development files are properly installed - - Rebuild from scratch after configuring with SSL support +## 🆘 Troubleshooting -2. If you experience SSL connection issues: - - You can specify certificate verification settings with environment variables: - ```bash - PGSSLMODE=verify-ca PGSSLROOTCERT=/path/to/rootcert.pem ./src/bin/psql/psql --dsql ... - ``` - -### General Build Issues - -If you encounter library dependency issues: - -1. Ensure the library path is correctly set to point to your built libpq and aws-dsql-auth libraries -2. Check for missing dependencies with `ldd` (Linux) or `otool -L` (macOS) -3. Verify that the configure step completed without errors - -### DSQL Authentication Issues - -If you encounter issues with DSQL authentication: - -1. Ensure your AWS credentials are correctly configured and have appropriate permissions -2. Check that you're using the correct endpoint hostname -3. Verify the `aws-dsql-auth` library is correctly built -4. Try with explicit AWS credentials via environment variables: - ```bash - AWS_ACCESS_KEY_ID=your_key AWS_SECRET_ACCESS_KEY=your_secret ./src/bin/psql/psql --dsql ... - ``` +### Connection Issues -## Notes +**Error: "could not connect to server"** +- Verify your DSQL endpoint URL is correct +- Ensure your security group allows connections on port 5432 +- Check that your AWS credentials are properly configured -- This builds only the command line client, not the PostgreSQL server -- The built psql can connect to any compatible PostgreSQL server -- Command line editing requires readline support -- SSL support is required for DSQL Authentication to work properly -- DSQL Authentication is primarily intended for use with AWS Database Services for PostgreSQL -- Token generation happens automatically and is transparent to the user -- Currently, admin token generation is only supported for the `admin` user +### Authentication Issues -## Packaging and Distribution +**Error: "authentication failed"** +- Verify your AWS credentials have the necessary DSQL permissions +- Ensure you're connecting to the correct DSQL workgroup +- Check that your IAM user/role has `dsql:DbConnect` permissions -PostgreSQL DSQL client can be packaged into a distributable format that can be easily downloaded from GitHub Releases. +### AWS Credentials -### Overview of the Packaging System +**Error: "Unable to locate credentials"** +- Run `aws configure` to set up credentials +- Or set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables +- Verify your credentials work: `aws sts get-caller-identity` -The packaging system: +### Getting Help -- Takes the built `psql` binary and `libpq` library -- Copies them into a structured directory called `postgres-dsql` -- Renames `psql` to `pdsql` -- Sets up the proper relative library paths so `pdsql` can find `libpq` -- Creates a ZIP archive for distribution - -### Directory Structure - -The packaged distribution has the following structure: - -``` -postgres-dsql/ -├── bin/ -│ └── pdsql # Renamed psql binary -└── lib/ - └── libpq.5.dylib # PostgreSQL client library +For additional help: +```bash +pdsql --help ``` -### GitHub Actions Workflow - -The GitHub Actions workflow automates the build and packaging process, and is triggered: - -- Manually via workflow_dispatch -- On creation of version tags (v*) - -To create a new release: -1. Tag a commit with a version number: `git tag v1.0.0` -2. Push the tag: `git push origin v1.0.0` +## 🔄 Updates -The workflow will: -1. Build PostgreSQL with DSQL support -2. Package the binaries -3. Test the package -4. Upload the package as a GitHub release artifact - -### Local Development and Testing - -#### Testing the Packaging Locally - -To test the packaging process locally: +To update to the latest version, simply run the installation command again: ```bash -# Build if needed and package -./scripts/test-packaging.sh +curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/main/scripts/install.sh | sh ``` -This script will: -1. Build DSQL if not already built -2. Package the build into postgres-dsql.zip -3. Extract and test the binary by running `dpsql --version` +## 🏗️ Building from Source -#### Manual Testing +If you need to build from source or contribute to development, see our [Development Guide](README_PACKAGING.md) for detailed build instructions. -You can also test manually: +### Quick Build ```bash -# Build DSQL +git clone https://github.com/marcbowes/postgres.git +cd postgres +git submodule update --init --recursive ./scripts/build-dsql.sh - -# Package the build -./scripts/package.sh - -# Test the packaged binary -unzip -o postgres-dsql.zip -d /tmp -/tmp/postgres-dsql/bin/pdsql --version ``` -### Using the Packaged Binary +## 📄 License -After downloading and extracting the package: +This project is based on PostgreSQL and maintains compatibility with the PostgreSQL license. -```bash -unzip postgres-dsql.zip -cd postgres-dsql -./bin/pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres -``` +## 🤝 Contributing -Note: The renamed binary `pdsql` automatically enables DSQL authentication mode, so the `--dsql` flag is not necessary. +Contributions are welcome! Please see our contributing guidelines and feel free to submit issues or pull requests. -### Current Limitations +--- -- This packaging solution currently only supports ARM macOS -- The package assumes the necessary AWS credentials are available in the environment +**Note**: This tool is specifically designed for AWS DSQL connections. For regular PostgreSQL connections, use the standard `psql` client. From 36c173fb75ce507ae6090a2afe6e249645d1ded2 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:13:37 -0700 Subject: [PATCH 28/42] Relocate readme --- README_DSQL.md => .github/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README_DSQL.md => .github/README.md (100%) diff --git a/README_DSQL.md b/.github/README.md similarity index 100% rename from README_DSQL.md rename to .github/README.md From 4ec85c19dc3bb05667fcff131f16c825d4ac724f Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:41:36 -0700 Subject: [PATCH 29/42] direct, verify-full by default --- src/bin/psql/help.c | 4 ++-- src/bin/psql/startup.c | 1 - src/interfaces/libpq/fe-connect.c | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c index 5c8f9f03093ba..e43f90982137e 100644 --- a/src/bin/psql/help.c +++ b/src/bin/psql/help.c @@ -57,9 +57,9 @@ usage(unsigned short int pager) */ initPQExpBuffer(&buf); - HELP0("psql is the PostgreSQL interactive terminal.\n\n"); + HELP0("psql (with DSQL support) is the PostgreSQL interactive terminal.\n\n"); HELP0("Usage:\n"); - HELP0(" psql [OPTION]... [DBNAME [USERNAME]]\n\n"); + HELP0(" pdsql [OPTION]... [DBNAME [USERNAME]]\n\n"); HELP0("General options:\n"); HELP0(" -c, --command=COMMAND run only single command (SQL or internal) and exit\n"); diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index e3300224f5791..291fd317b024f 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -219,7 +219,6 @@ main(int argc, char *argv[]) if (pset.dsql) { - setenv("PGSSLMODE", "require", 0); setenv("PGDSQL", "1", 1); pset.getPassword = TRI_NO; } diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index be5d5254e5ec5..702b94c48db59 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -281,11 +281,11 @@ static const internalPQconninfoOption PQconninfoOptions[] = { * parameters have no effect on non-SSL connections, so there is no reason * to exclude them since none of them are mandatory. */ - {"sslmode", "PGSSLMODE", DefaultSSLMode, NULL, + {"sslmode", "PGSSLMODE", "verify-full", NULL, "SSL-Mode", "", 12, /* sizeof("verify-full") == 12 */ offsetof(struct pg_conn, sslmode)}, - {"sslnegotiation", "PGSSLNEGOTIATION", DefaultSSLNegotiation, NULL, + {"sslnegotiation", "PGSSLNEGOTIATION", "direct", NULL, "SSL-Negotiation", "", 9, /* sizeof("postgres") == 9 */ offsetof(struct pg_conn, sslnegotiation)}, @@ -309,7 +309,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "SSL-Client-Key-Password", "*", 20, offsetof(struct pg_conn, sslpassword)}, - {"sslrootcert", "PGSSLROOTCERT", NULL, NULL, + {"sslrootcert", "PGSSLROOTCERT", "system", NULL, "SSL-Root-Certificate", "", 64, offsetof(struct pg_conn, sslrootcert)}, From c932f2ec18372712788081fde8b131469fae80d7 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:55:02 -0700 Subject: [PATCH 30/42] Fix install link --- .github/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/README.md b/.github/README.md index 83cc1091f468c..2a06f9a525486 100644 --- a/.github/README.md +++ b/.github/README.md @@ -7,7 +7,7 @@ A PostgreSQL command-line client with built-in AWS DSQL authentication support. Install `pdsql` with a single command: ```bash -curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/main/scripts/install.sh | sh +curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/refs/heads/master/scripts/install.sh | sh ``` This installer automatically detects your platform (macOS/Linux) and architecture, downloads the appropriate package, and installs `pdsql` to your local environment. @@ -176,7 +176,7 @@ pdsql --help To update to the latest version, simply run the installation command again: ```bash -curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/main/scripts/install.sh | sh +curl -sSL https://raw.githubusercontent.com/marcbowes/postgres/refs/heads/master/scripts/install.sh | sh ``` ## 🏗️ Building from Source From e4a5eb921b8356fd96548cb9ff3c21ca271b9e28 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Sat, 21 Jun 2025 21:55:09 -0700 Subject: [PATCH 31/42] Default db to postgres --- src/interfaces/libpq/fe-connect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 702b94c48db59..682b3b064c406 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -223,7 +223,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Connect-timeout", "", 10, /* strlen(INT32_MAX) == 10 */ offsetof(struct pg_conn, connect_timeout)}, - {"dbname", "PGDATABASE", NULL, NULL, + {"dbname", "PGDATABASE", "postgres", NULL, "Database-Name", "", 20, offsetof(struct pg_conn, dbName)}, From 060acad30a8b9ee04d310e44bc2d788400f6c759 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Wed, 25 Jun 2025 14:53:25 -0700 Subject: [PATCH 32/42] Exclude libss/crypto from libpq On linux, we have the following dep chain: aws-dsql-auth -> aws-c-http -> aws-c-io -> aws-lc We only use aws-c-http for request signing, but we cannot make aws-c-io conditionally required. aws-c-io only works with aws-lc on linux, which means we land up in a situation where we have two libcryptos (postgres wants openssl). Our overall solution is to make fe-dsql-auth.o a self-contained file (hiding the aws-dsql-auth dep chain) and explicitly exclude the libs we don't want. --- src/interfaces/libpq/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 1d0b9eee226ed..4e016309f34ed 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -81,7 +81,7 @@ OBJS += \ endif AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) -AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') +AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*' -not -name libssl.a -not -name libcrypto.a) # Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c From 2a8eaec4b39d5acfd6273f32e72577c23cf2d779 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:49:44 -0700 Subject: [PATCH 33/42] pbbench support for dsql Doesn't handle occ yet --- scripts/build-dsql.sh | 19 ++++++- src/bin/pgbench/pgbench.c | 109 +++++++++++++++++++++++++++++--------- 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 6deb2136aedc4..39fc3101436e6 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -81,10 +81,14 @@ echo "Step 4: Building psql..." make -C src/bin/psql echo " psql built successfully!" +# Step 5: Build pgbench +echo "Step 5: Building pgbench..." +make -C src/bin/pgbench +echo " pgbench built successfully!" + # Final instructions echo "" echo "Build completed successfully!" -echo "" echo "To run psql with DSQL authentication, use the following command:" echo "" echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" @@ -95,4 +99,17 @@ echo "" echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" echo " ./src/bin/psql/psql --dsql \"dbname=postgres user=admin host=your-dsql-endpoint.example.com\"" echo "" +echo "To run pgbench with DSQL authentication, use the following command:" +echo "" +echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" +echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "" +echo "For example, to run a simple benchmark test:" +echo "" +echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" +echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --initialize --scale=1" +echo "" +echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" +echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --time=60 --client=10" +echo "" echo "Note: You need to have AWS credentials configured in your environment for DSQL authentication to work." diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 497a936c141f3..8c4617917bb34 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -771,6 +771,8 @@ static bool verbose_errors = false; /* print verbose messages of all errors */ static bool exit_on_abort = false; /* exit when any client is aborted */ +static bool dsql = false; /* --dsql command line option */ + /* Builtin test scripts */ typedef struct BuiltinScript { @@ -849,6 +851,10 @@ static const PsqlScanCallbacks pgbench_callbacks = { NULL, /* don't need get_variable functionality */ }; +static bool is_dsql() { + return strcmp(getenv("PGDSQL"), "1") == 0; +} + static char get_table_relkind(PGconn *con, const char *table) { @@ -4852,26 +4858,26 @@ initCreateTables(PGconn *con) static const struct ddlinfo DDLs[] = { { "pgbench_history", - "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)", - "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)", + "id uuid default gen_random_uuid(),tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)", + "id uuid default gen_random_uuid(),tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)", 0 }, { "pgbench_tellers", - "tid int not null,bid int,tbalance int,filler char(84)", - "tid int not null,bid int,tbalance int,filler char(84)", + "tid int primary key,bid int,tbalance int,filler char(84)", + "tid int primary key,bid int,tbalance int,filler char(84)", 1 }, { "pgbench_accounts", - "aid int not null,bid int,abalance int,filler char(84)", - "aid bigint not null,bid int,abalance int,filler char(84)", + "aid int primary key,bid int,abalance int,filler char(84)", + "aid bigint primary key,bid int,abalance int,filler char(84)", 1 }, { "pgbench_branches", - "bid int not null,bbalance int,filler char(88)", - "bid int not null,bbalance int,filler char(88)", + "bid int primary key,bbalance int,filler char(88)", + "bid int primary key,bbalance int,filler char(88)", 1 } }; @@ -4892,14 +4898,16 @@ initCreateTables(PGconn *con) ddl->table, (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols); - /* Partition pgbench_accounts table */ - if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0) - appendPQExpBuffer(&query, - " partition by %s (aid)", PARTITION_METHOD[partition_method]); - else if (ddl->declare_fillfactor) - { - /* fillfactor is only expected on actual tables */ - appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor); + if (!is_dsql()) { + /* Partition pgbench_accounts table */ + if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0) + appendPQExpBuffer(&query, + " partition by %s (aid)", PARTITION_METHOD[partition_method]); + else if (ddl->declare_fillfactor) + { + /* fillfactor is only expected on actual tables */ + appendPQExpBuffer(&query, " with (fillfactor=%d)", fillfactor); + } } if (tablespace != NULL) @@ -4926,11 +4934,17 @@ initCreateTables(PGconn *con) static void initTruncateTables(PGconn *con) { - executeStatement(con, "truncate table " + if (!is_dsql()) { + executeStatement(con, "truncate table " "pgbench_accounts, " "pgbench_branches, " "pgbench_history, " "pgbench_tellers"); + } else { + // TODO: DSQL mode needs to do deletes in batches. + // This is not implemented because it's relatively easy to simply + // drop all tables and re-init with -i. + } } static void @@ -4984,8 +4998,9 @@ initPopulateTable(PGconn *con, const char *table, int64 base, initPQExpBuffer(&sql); /* Use COPY with FREEZE on v14 and later for all ordinary tables */ + // XXX: DSQL doesn't support (or need) freeze. if ((PQserverVersion(con) >= 140000) && - get_table_relkind(con, table) == RELKIND_RELATION) + get_table_relkind(con, table) == RELKIND_RELATION && !is_dsql()) copy_statement_fmt = "copy %s from stdin with (freeze on)"; @@ -4995,16 +5010,22 @@ initPopulateTable(PGconn *con, const char *table, int64 base, else if (n == -1) pg_fatal("invalid format string"); - res = PQexec(con, copy_statement); - - if (PQresultStatus(res) != PGRES_COPY_IN) - pg_fatal("unexpected copy in result: %s", PQerrorMessage(con)); - PQclear(res); - start = pg_time_now(); for (k = 0; k < total; k++) { + if (k == 0) { + if (is_dsql()) { + executeStatement(con, "begin"); + } + + res = PQexec(con, copy_statement); + + if (PQresultStatus(res) != PGRES_COPY_IN) + pg_fatal("unexpected copy in result: %s", PQerrorMessage(con)); + PQclear(res); + } + int64 j = k + 1; init_row(&sql, k); @@ -5014,6 +5035,21 @@ initPopulateTable(PGconn *con, const char *table, int64 base, if (CancelRequested) break; + // XXX: Start a new transaction every 1000 rows + if (is_dsql() && (k > 0 && k % 1000 == 0)) { + if (PQputline(con, "\\.\n")) + pg_fatal("very last PQputline failed"); + if (PQendcopy(con)) + pg_fatal("PQendcopy failed"); + + executeStatement(con, "commit"); + executeStatement(con, "begin"); + res = PQexec(con, copy_statement); + if (PQresultStatus(res) != PGRES_COPY_IN) + pg_fatal("unexpected copy in result: %s", PQerrorMessage(con)); + PQclear(res); + } + /* * If we want to stick with the original logging, print a message each * 100k inserted rows. @@ -5076,6 +5112,10 @@ initPopulateTable(PGconn *con, const char *table, int64 base, if (PQendcopy(con)) pg_fatal("PQendcopy failed"); + if (is_dsql()) { + executeStatement(con, "commit"); + } + termPQExpBuffer(&sql); } @@ -5090,11 +5130,14 @@ initGenerateDataClientSide(PGconn *con) { fprintf(stderr, "generating data (client-side)...\n"); + // XXX: On DSQL we do batch inserts within populate. + if (!is_dsql()) { /* * we do all of this in one transaction to enable the backend's * data-loading optimizations */ executeStatement(con, "begin"); + } /* truncate away any old data */ initTruncateTables(con); @@ -5107,7 +5150,9 @@ initGenerateDataClientSide(PGconn *con) initPopulateTable(con, "pgbench_tellers", ntellers, initTeller); initPopulateTable(con, "pgbench_accounts", naccounts, initAccount); + if (!is_dsql()) { executeStatement(con, "commit"); + } } /* @@ -5178,6 +5223,11 @@ initVacuum(PGconn *con) static void initCreatePKeys(PGconn *con) { + // Schema has been updated to always have PKs. + if (is_dsql()) { + return; + } + static const char *const DDLINDEXes[] = { "alter table pgbench_branches add primary key (bid)", "alter table pgbench_tellers add primary key (tid)", @@ -6705,6 +6755,7 @@ main(int argc, char **argv) {"verbose-errors", no_argument, NULL, 15}, {"exit-on-abort", no_argument, NULL, 16}, {"debug", no_argument, NULL, 17}, + {"dsql", no_argument, NULL, 18}, {NULL, 0, NULL, 0} }; @@ -7058,6 +7109,9 @@ main(int argc, char **argv) case 17: /* debug */ pg_logging_increase_verbosity(); break; + case 18: /* dsql */ + dsql = true; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -7065,6 +7119,13 @@ main(int argc, char **argv) } } + if (dsql) + { + setenv("PGDSQL", "1", 1); + is_no_vacuum = true; + foreign_keys = false; + } + /* set default script if none */ if (num_scripts == 0 && !is_init_mode) { From 6e4bdb319adb3261c4e0c7027a0545ff2307bedc Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Thu, 26 Jun 2025 16:11:01 -0700 Subject: [PATCH 34/42] pkg pgbench --- scripts/build-dsql.sh | 6 ++-- scripts/package.sh | 77 +++++++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 39fc3101436e6..0c225191ae4c9 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -102,14 +102,14 @@ echo "" echo "To run pgbench with DSQL authentication, use the following command:" echo "" echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" -echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo " ./src/bin/pgbench/pgbench --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" echo "" echo "For example, to run a simple benchmark test:" echo "" echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" -echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --initialize --scale=1" +echo " ./src/bin/pgbench/pgbench --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --initialize --scale=1" echo "" echo " $LIBRARY_PATH_VAR=$(pwd)/src/interfaces/libpq \\" -echo " ./src/bin/pgbench/pgbench --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --time=60 --client=10" +echo " ./src/bin/pgbench/pgbench --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --time=60 --client=10" echo "" echo "Note: You need to have AWS credentials configured in your environment for DSQL authentication to work." diff --git a/scripts/package.sh b/scripts/package.sh index 851dd8ebf5c9a..262b88dd67ddd 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -2,8 +2,8 @@ set -e # Package PostgreSQL DSQL client for distribution -# This script creates a standalone distribution with psql (renamed to pdsql) -# and libpq that can be used without additional dependencies +# This script creates a standalone distribution with psql (renamed to pdsql), +# pgbench, and libpq that can be used without additional dependencies echo "Packaging PostgreSQL DSQL client" echo "================================" @@ -13,8 +13,10 @@ ROOT_DIR=$(pwd) BUILD_DIR="$ROOT_DIR/build" DIST_NAME="postgres-dsql" DIST_DIR="$BUILD_DIR/$DIST_NAME" -SRC_BIN="$ROOT_DIR/src/bin/psql/psql" -BINARY_NAME="pdsql" +SRC_PSQL_BIN="$ROOT_DIR/src/bin/psql/psql" +SRC_PGBENCH_BIN="$ROOT_DIR/src/bin/pgbench/pgbench" +PSQL_BINARY_NAME="pdsql" +PGBENCH_BINARY_NAME="pgbench" # Detect OS and set appropriate library paths if [[ "$OSTYPE" == "darwin"* ]]; then @@ -34,8 +36,14 @@ else fi # Check if the build artifacts exist -if [ ! -f "$SRC_BIN" ]; then - echo "Error: psql binary not found at $SRC_BIN" +if [ ! -f "$SRC_PSQL_BIN" ]; then + echo "Error: psql binary not found at $SRC_PSQL_BIN" + echo "Please run scripts/build-dsql.sh first" + exit 1 +fi + +if [ ! -f "$SRC_PGBENCH_BIN" ]; then + echo "Error: pgbench binary not found at $SRC_PGBENCH_BIN" echo "Please run scripts/build-dsql.sh first" exit 1 fi @@ -60,8 +68,11 @@ mkdir -p "$DIST_DIR/bin" mkdir -p "$DIST_DIR/lib" # Copy binaries and libraries -echo "Copying psql to $DIST_DIR/bin/$BINARY_NAME" -cp "$SRC_BIN" "$DIST_DIR/bin/$BINARY_NAME" +echo "Copying psql to $DIST_DIR/bin/$PSQL_BINARY_NAME" +cp "$SRC_PSQL_BIN" "$DIST_DIR/bin/$PSQL_BINARY_NAME" + +echo "Copying pgbench to $DIST_DIR/bin/$PGBENCH_BINARY_NAME" +cp "$SRC_PGBENCH_BIN" "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" echo "Copying libpq to $DIST_DIR/lib/" cp "$SRC_LIB" "$DIST_DIR/lib/" @@ -71,41 +82,52 @@ if [[ "$PLATFORM" == "macos" ]]; then # Copy additional dylib if it exists cp "$ROOT_DIR/src/interfaces/libpq/libpq.dylib" "$DIST_DIR/lib/" 2>/dev/null || true - # Set up correct library paths in the binary - echo "Updating library paths in $BINARY_NAME binary..." - LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq | awk '{print $1}') - install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$BINARY_NAME" + # Set up correct library paths in the binaries + echo "Updating library paths in $PSQL_BINARY_NAME binary..." + LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$PSQL_BINARY_NAME" | grep libpq | awk '{print $1}') + install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$PSQL_BINARY_NAME" + + echo "Updating library paths in $PGBENCH_BINARY_NAME binary..." + LIBRARY_PATH=$(otool -L "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" | grep libpq | awk '{print $1}') + install_name_tool -change "$LIBRARY_PATH" "@loader_path/../lib/libpq.5.dylib" "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" # Fix library itself to refer to itself by relative path install_name_tool -id "@loader_path/libpq.5.dylib" "$DIST_DIR/lib/libpq.5.dylib" # Verify the changes echo "Verifying library path changes:" - otool -L "$DIST_DIR/bin/$BINARY_NAME" | grep libpq + otool -L "$DIST_DIR/bin/$PSQL_BINARY_NAME" | grep libpq + otool -L "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" | grep libpq otool -L "$DIST_DIR/lib/libpq.5.dylib" | grep libpq elif [[ "$PLATFORM" == "linux" ]]; then # Copy additional .so files if they exist cp "$ROOT_DIR/src/interfaces/libpq/libpq.so" "$DIST_DIR/lib/" 2>/dev/null || true - # Set up RPATH for the binary to find libraries in ../lib - echo "Setting RPATH for $BINARY_NAME binary..." - patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" 2>/dev/null || { + # Set up RPATH for the binaries to find libraries in ../lib + echo "Setting RPATH for $PSQL_BINARY_NAME binary..." + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PSQL_BINARY_NAME" 2>/dev/null || { echo "Warning: patchelf not available. Installing patchelf..." if command -v apt-get >/dev/null 2>&1; then sudo apt-get update && sudo apt-get install -y patchelf - patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PSQL_BINARY_NAME" + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" elif command -v yum >/dev/null 2>&1; then sudo yum install -y patchelf - patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$BINARY_NAME" + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PSQL_BINARY_NAME" + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" else echo "Warning: Could not install patchelf. The binary may not find libraries correctly." fi } + echo "Setting RPATH for $PGBENCH_BINARY_NAME binary..." + patchelf --set-rpath '$ORIGIN/../lib' "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" 2>/dev/null || echo "patchelf already handled above" + # Verify the changes echo "Verifying RPATH changes:" - ldd "$DIST_DIR/bin/$BINARY_NAME" | grep libpq || echo "libpq dependency check complete" + ldd "$DIST_DIR/bin/$PSQL_BINARY_NAME" | grep libpq || echo "libpq dependency check complete for pdsql" + ldd "$DIST_DIR/bin/$PGBENCH_BINARY_NAME" | grep libpq || echo "libpq dependency check complete for pgbench" fi # Create a ZIP archive @@ -171,20 +193,27 @@ mkdir -p %{buildroot}/usr/bin # Install binaries and libraries to /opt to avoid conflicts cp %{_sourcedir}/bin/pdsql %{buildroot}/opt/postgres-dsql/bin/ +cp %{_sourcedir}/bin/pgbench %{buildroot}/opt/postgres-dsql/bin/ cp %{_sourcedir}/lib/* %{buildroot}/opt/postgres-dsql/lib/ -# Create symlink in /usr/bin for easy access +# Create symlinks in /usr/bin for easy access ln -s /opt/postgres-dsql/bin/pdsql %{buildroot}/usr/bin/pdsql +ln -s /opt/postgres-dsql/bin/pgbench %{buildroot}/usr/bin/pgbench %files /opt/postgres-dsql/bin/pdsql +/opt/postgres-dsql/bin/pgbench /opt/postgres-dsql/lib/* /usr/bin/pdsql +/usr/bin/pgbench %post echo "PostgreSQL DSQL client installed successfully!" echo "Use 'pdsql' command to connect to AWS DSQL databases." echo "Example: pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "" +echo "Use 'pgbench' command to run PostgreSQL benchmarks against DSQL databases." +echo "Example: pgbench --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --initialize --scale=1" %changelog * $(date +'%a %b %d %Y') Build System - 1.0.0-1 @@ -195,6 +224,7 @@ EOF mkdir -p "$RPM_BUILD_DIR/SOURCES/bin" mkdir -p "$RPM_BUILD_DIR/SOURCES/lib" cp "$DIST_DIR/bin/pdsql" "$RPM_BUILD_DIR/SOURCES/bin/" + cp "$DIST_DIR/bin/pgbench" "$RPM_BUILD_DIR/SOURCES/bin/" cp "$DIST_DIR/lib"/* "$RPM_BUILD_DIR/SOURCES/lib/" # Build the RPM @@ -271,10 +301,12 @@ EOF # Copy files cp "$DIST_DIR/bin/pdsql" "$DEB_PKG_DIR/opt/postgres-dsql/bin/" + cp "$DIST_DIR/bin/pgbench" "$DEB_PKG_DIR/opt/postgres-dsql/bin/" cp "$DIST_DIR/lib"/* "$DEB_PKG_DIR/opt/postgres-dsql/lib/" - # Create symlink + # Create symlinks ln -s /opt/postgres-dsql/bin/pdsql "$DEB_PKG_DIR/usr/bin/pdsql" + ln -s /opt/postgres-dsql/bin/pgbench "$DEB_PKG_DIR/usr/bin/pgbench" # Create control file cat > "$DEB_PKG_DIR/DEBIAN/control" << EOF @@ -297,6 +329,9 @@ EOF echo "PostgreSQL DSQL client installed successfully!" echo "Use 'pdsql' command to connect to AWS DSQL databases." echo "Example: pdsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres" +echo "" +echo "Use 'pgbench' command to run PostgreSQL benchmarks against DSQL databases." +echo "Example: pgbench --dsql --host=your-dsql-endpoint.example.com --user=admin --dbname=postgres --initialize --scale=1" EOF chmod 755 "$DEB_PKG_DIR/DEBIAN/postinst" From 0187de5adff741e7f6810eaa4eaa7eb41dd1e784 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Thu, 26 Jun 2025 22:47:38 -0700 Subject: [PATCH 35/42] Revert the tls filter It doesn't work because we end up with a few unnamed syms. --- src/interfaces/libpq/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 4e016309f34ed..1d0b9eee226ed 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -81,7 +81,7 @@ OBJS += \ endif AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) -AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*' -not -name libssl.a -not -name libcrypto.a) +AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') # Custom rule for fe-dsql-auth.o that includes AWS libraries fe-dsql-auth.o: fe-dsql-auth.c From d61f60f5b9a0588b92694e22f2e9980e1d0abbc6 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:54:05 -0700 Subject: [PATCH 36/42] Bump aws-dsql-auth --- aws-dsql-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-dsql-auth b/aws-dsql-auth index 86a5f98749538..f33fce18f3864 160000 --- a/aws-dsql-auth +++ b/aws-dsql-auth @@ -1 +1 @@ -Subproject commit 86a5f9874953866409095f3f2391596ec51dd1a9 +Subproject commit f33fce18f3864e9428b10ae63c018b004b18436a From b501e2abe06135a7cc77c3d309bf40daeeb19b45 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 17:06:36 -0700 Subject: [PATCH 37/42] Use sym viz. to deal with dups --- scripts/build-dsql.sh | 2 +- src/interfaces/libpq/Makefile | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 0c225191ae4c9..83173df1b8ada 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -51,7 +51,7 @@ else echo " aws-dsql-auth submodules already initialized." fi -if [ ! -f "aws-dsql-auth/build/aws-dsql-auth/libaws-dsql-auth.a" ]; then +if [ ! -f "aws-dsql-auth/build/install/lib64/libaws-dsql-auth.a" ]; then # Build aws-dsql-auth echo " Building aws-dsql-auth library..." cd aws-dsql-auth diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 1d0b9eee226ed..03556fcd12c49 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -80,18 +80,31 @@ OBJS += \ win32.o endif -AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/ -type d -name include -not -path '*/test*')) -AWS_DSQL_AUTH_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/ -type f -name '*.a' -not -path '*/test*') +AWS_DSQL_AUTH_CPPFLAGS := $(addprefix -I,$(shell find $(top_srcdir)/aws-dsql-auth/build/install -type d -name include)) +AWS_DSQL_AUTH_ALL_LIBS := $(shell find $(top_srcdir)/aws-dsql-auth/build/install/ -type f -name '*.a') -# Custom rule for fe-dsql-auth.o that includes AWS libraries +# Custom rule for fe-dsql-auth.o that hides all symbols except the public API fe-dsql-auth.o: fe-dsql-auth.c $(CC) $(CFLAGS) $(CPPFLAGS) $(AWS_DSQL_AUTH_CPPFLAGS) -c fe-dsql-auth.c -o fe-dsql-auth-temp.o ifeq ($(PORTNAME), linux) - $(LD) -r -o $@ fe-dsql-auth-temp.o --whole-archive $(AWS_DSQL_AUTH_LIBS) + # Link with AWS libraries using start-group to resolve dependencies + $(LD) -r -o fe-dsql-auth-with-aws.o fe-dsql-auth-temp.o --start-group $(AWS_DSQL_AUTH_ALL_LIBS) --end-group + # Create a list of symbols to keep (only the public API from fe-dsql-auth.h) + echo "generate_dsql_token" > keep-symbols.txt + echo "dsql_auth_cleanup" >> keep-symbols.txt + # Hide all symbols except the ones we want to keep + objcopy --keep-global-symbols=keep-symbols.txt fe-dsql-auth-with-aws.o $@ + rm -f fe-dsql-auth-temp.o fe-dsql-auth-with-aws.o keep-symbols.txt +else ifeq ($(PORTNAME), darwin) + # macOS: Use ld with exported symbols list + echo "_generate_dsql_token" > exported-symbols.txt + echo "_dsql_auth_cleanup" >> exported-symbols.txt + $(LD) -r -o $@ fe-dsql-auth-temp.o -exported_symbols_list exported-symbols.txt $(AWS_DSQL_AUTH_ALL_LIBS) + rm -f fe-dsql-auth-temp.o exported-symbols.txt else - $(LD) -r -o $@ fe-dsql-auth-temp.o $(AWS_DSQL_AUTH_LIBS) -endif + $(LD) -r -o $@ fe-dsql-auth-temp.o --whole-archive $(AWS_DSQL_AUTH_ALL_LIBS) rm -f fe-dsql-auth-temp.o +endif # Add libraries that libpq depends (or might depend) on into the # shared library link. (The order in which you list them here doesn't From 560789dfe23ba0b03a0ee3555f64b36de5639537 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 20:41:06 -0700 Subject: [PATCH 38/42] Add logging and imds support This commit wires up the aws-c-common logging infra. You can log to std{out,err} or a file, and set the level through env vars. The readme has examples. With this logging I realized that imds wasn't working due to a missing http client. That's been fixed now. --- .github/README.md | 62 ++++++++++++ src/interfaces/libpq/fe-dsql-auth.c | 144 +++++++++++++++++++++++++--- 2 files changed, 195 insertions(+), 11 deletions(-) diff --git a/.github/README.md b/.github/README.md index 2a06f9a525486..4f9cb6e604ac1 100644 --- a/.github/README.md +++ b/.github/README.md @@ -164,6 +164,68 @@ pdsql --host=your-endpoint.dsql.amazonaws.com \ - Or set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables - Verify your credentials work: `aws sts get-caller-identity` +## 🔍 Debug Logging + +For troubleshooting authentication and connection issues, enable detailed AWS SDK logging: + +### Environment Variables + +- **`AWS_LOG_LEVEL`**: Controls verbosity (NONE, FATAL, ERROR, WARN, INFO, DEBUG, TRACE) +- **`AWS_LOG_FILE`**: Controls output destination (stdout, stderr, or file path) + +### Basic Debugging + +Enable debug logging to stderr (default): +```bash +AWS_LOG_LEVEL=DEBUG pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin +``` + +### Detailed Tracing + +Enable maximum verbosity for deep debugging: +```bash +AWS_LOG_LEVEL=TRACE pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin +``` + +### Log to File + +Save logs to a file for analysis: +```bash +AWS_LOG_LEVEL=DEBUG AWS_LOG_FILE=/tmp/dsql-debug.log pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin +``` + +### Log to stdout + +Send logs to stdout (useful for piping): +```bash +AWS_LOG_LEVEL=INFO AWS_LOG_FILE=stdout pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin +``` + +### What the Logs Show + +The debug logs will reveal: +- **Token Generation**: Process of creating DSQL authentication tokens +- **AWS Region Detection**: How the region is determined from hostname or environment +- **Credentials Provider Chain**: Which credential sources are tried (environment, files, IAM roles, IMDS) +- **HTTP Infrastructure**: Event loops and network setup for IMDS on EC2 +- **Error Details**: Specific AWS SDK errors with error codes + +### Example Log Output + +``` +[INFO] [2025-06-28T03:33:55Z] Starting DSQL token generation for endpoint: your-endpoint.dsql.amazonaws.com +[DEBUG] [2025-06-28T03:33:55Z] Using AWS_REGION from environment: us-west-2 +[DEBUG] [2025-06-28T03:33:55Z] Creating credentials provider chain with bootstrap for IMDS +[INFO] [2025-06-28T03:33:55Z] Token generation successful +``` + +### Disable Logging + +To disable all logging: +```bash +AWS_LOG_LEVEL=NONE pdsql --host=your-endpoint.dsql.amazonaws.com --user=admin +``` + ### Getting Help For additional help: diff --git a/src/interfaces/libpq/fe-dsql-auth.c b/src/interfaces/libpq/fe-dsql-auth.c index 217e19c1c9c4f..c7ed75d300eaf 100644 --- a/src/interfaces/libpq/fe-dsql-auth.c +++ b/src/interfaces/libpq/fe-dsql-auth.c @@ -17,27 +17,76 @@ /* Include AWS DSQL Auth library functions */ #include +#include #include #include #include +#include +#include +#include +#include #include #include static bool aws_libs_initialized = false; +static struct aws_logger dsql_logger; +static bool dsql_logger_initialized = false; + +/* HTTP infrastructure for IMDS */ +static struct aws_event_loop_group *s_el_group = NULL; +static struct aws_host_resolver *s_host_resolver = NULL; +static struct aws_client_bootstrap *s_client_bootstrap = NULL; /* - * This function can be called to clean up AWS library resources + * Initialize DSQL logging */ static void -_dsql_auth_cleanup(void) +initialize_dsql_logging(void) { - if (aws_libs_initialized) + if (!dsql_logger_initialized) { - aws_sdkutils_library_clean_up(); - aws_auth_library_clean_up(); - aws_io_library_clean_up(); - aws_common_library_clean_up(); - aws_libs_initialized = false; + struct aws_allocator *allocator = aws_default_allocator(); + struct aws_logger_standard_options logger_options = { + .level = AWS_LOG_LEVEL_DEBUG, /* Can be controlled by environment variable */ + .file = stderr /* Log to stderr by default */ + }; + + /* Check for AWS_LOG_LEVEL environment variable */ + const char *log_level_str = getenv("AWS_LOG_LEVEL"); + if (log_level_str != NULL) + { + enum aws_log_level level; + if (aws_string_to_log_level(log_level_str, &level) == AWS_OP_SUCCESS) + { + logger_options.level = level; + } + } + + /* Check for AWS_LOG_FILE environment variable for file output */ + const char *log_file_str = getenv("AWS_LOG_FILE"); + if (log_file_str != NULL && strlen(log_file_str) > 0) + { + if (strcmp(log_file_str, "stdout") == 0) + { + logger_options.file = stdout; + } + else if (strcmp(log_file_str, "stderr") == 0) + { + logger_options.file = stderr; + } + else + { + /* Use as a filename */ + logger_options.filename = log_file_str; + logger_options.file = NULL; + } + } + + if (aws_logger_init_standard(&dsql_logger, allocator, &logger_options) == AWS_OP_SUCCESS) + { + aws_logger_set(&dsql_logger); + dsql_logger_initialized = true; + } } } @@ -47,7 +96,21 @@ _dsql_auth_cleanup(void) void dsql_auth_cleanup(void) { - _dsql_auth_cleanup(); + if (dsql_logger_initialized) + { + aws_logger_set(NULL); + aws_logger_clean_up(&dsql_logger); + dsql_logger_initialized = false; + } + + if (aws_libs_initialized) + { + aws_sdkutils_library_clean_up(); + aws_auth_library_clean_up(); + aws_io_library_clean_up(); + aws_common_library_clean_up(); + aws_libs_initialized = false; + } } /* @@ -61,13 +124,62 @@ initialize_aws_libs(void) struct aws_allocator *allocator = aws_default_allocator(); aws_common_library_init(allocator); aws_io_library_init(allocator); + aws_http_library_init(allocator); aws_auth_library_init(allocator); aws_sdkutils_library_init(allocator); + + /* Initialize HTTP infrastructure for IMDS */ + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Initializing HTTP infrastructure for IMDS"); + + s_el_group = aws_event_loop_group_new_default(allocator, 1, NULL); + if (!s_el_group) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create event loop group"); + goto error; + } + + struct aws_host_resolver_default_options resolver_options = { + .el_group = s_el_group, + .max_entries = 8, + }; + s_host_resolver = aws_host_resolver_new_default(allocator, &resolver_options); + if (!s_host_resolver) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create host resolver"); + goto error; + } + + struct aws_client_bootstrap_options bootstrap_options = { + .event_loop_group = s_el_group, + .host_resolver = s_host_resolver, + }; + s_client_bootstrap = aws_client_bootstrap_new(allocator, &bootstrap_options); + if (!s_client_bootstrap) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create client bootstrap"); + goto error; + } + aws_libs_initialized = true; + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "AWS libraries and HTTP infrastructure initialized successfully"); /* Note: We cannot use atexit() in libpq as it's not allowed to call exit-related functions. * The cleanup will be handled by explicit calls at application shutdown or by the OS. */ + return; + + error: + /* Clean up on error */ + if (s_client_bootstrap) { + aws_client_bootstrap_release(s_client_bootstrap); + s_client_bootstrap = NULL; + } + if (s_host_resolver) { + aws_host_resolver_release(s_host_resolver); + s_host_resolver = NULL; + } + if (s_el_group) { + aws_event_loop_group_release(s_el_group); + s_el_group = NULL; + } + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to initialize AWS libraries"); } } @@ -90,11 +202,14 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) const char *env_region; const char *token_str; - /* Initialize AWS libraries */ + /* Initialize AWS libraries and logging */ initialize_aws_libs(); + initialize_dsql_logging(); allocator = aws_default_allocator(); + AWS_LOGF_INFO(AWS_LS_AUTH_GENERAL, "Starting DSQL token generation for endpoint: %s", endpoint); + /* Initialize DSQL auth config */ if (aws_dsql_auth_config_init(&auth_config) != AWS_OP_SUCCESS) { if (err_msg) @@ -109,8 +224,10 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) env_region = getenv("AWS_REGION"); if (env_region != NULL && env_region[0] != '\0') { + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Using AWS_REGION from environment: %s", env_region); aws_region = aws_string_new_from_c_str(allocator, env_region); if (!aws_region) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create region string from AWS_REGION"); if (err_msg) *err_msg = strdup("Failed to create region string from AWS_REGION"); goto cleanup; @@ -118,20 +235,25 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) } else { + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "AWS_REGION not set, attempting to infer from hostname: %s", endpoint); /* Try to infer region from hostname */ if (aws_dsql_auth_config_infer_region(allocator, &auth_config, &aws_region) != AWS_OP_SUCCESS || aws_region == NULL) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to infer AWS region from hostname: %s", endpoint); if (err_msg) *err_msg = strdup("Failed to infer AWS region from hostname. Please set AWS_REGION environment variable."); goto cleanup; } + AWS_LOGF_INFO(AWS_LS_AUTH_GENERAL, "Inferred region: %s", aws_string_c_str(aws_region)); } aws_dsql_auth_config_set_region(&auth_config, aws_region); - /* Create default credentials provider */ + /* Create default credentials provider with client bootstrap for IMDS */ AWS_ZERO_STRUCT(credentials_options); + credentials_options.bootstrap = s_client_bootstrap; + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Creating credentials provider chain with bootstrap for IMDS"); credentials_provider = aws_credentials_provider_new_chain_default(allocator, &credentials_options); if (!credentials_provider) { aws_error = aws_last_error(); From 6c19265f447ed89bb2530fbbd6aaac15a4291569 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 22:26:59 -0700 Subject: [PATCH 39/42] Improve init The token gen infra (allocator, creds provider) are setup once, globally. This lets us cache creds and do a fast-fail if there are no creds. --- scripts/build-dsql.sh | 2 +- src/bin/pgbench/pgbench.c | 24 +++ src/bin/psql/startup.c | 26 +++ src/interfaces/libpq/Makefile | 12 +- src/interfaces/libpq/exports.txt | 4 + src/interfaces/libpq/fe-connect.c | 2 +- src/interfaces/libpq/fe-dsql-auth.c | 321 +++++++++++++++++++++------- src/interfaces/libpq/fe-dsql-auth.h | 10 +- 8 files changed, 320 insertions(+), 81 deletions(-) diff --git a/scripts/build-dsql.sh b/scripts/build-dsql.sh index 83173df1b8ada..2a2f9f6197cab 100755 --- a/scripts/build-dsql.sh +++ b/scripts/build-dsql.sh @@ -51,7 +51,7 @@ else echo " aws-dsql-auth submodules already initialized." fi -if [ ! -f "aws-dsql-auth/build/install/lib64/libaws-dsql-auth.a" ]; then +if [ ! -d "aws-dsql-auth/build/install/" ]; then # Build aws-dsql-auth echo " Building aws-dsql-auth library..." cd aws-dsql-auth diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 8c4617917bb34..967612a185828 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -68,6 +68,7 @@ #include "pgbench.h" #include "port/pg_bitutils.h" #include "portability/instr_time.h" +#include "fe-dsql-auth.h" /* X/Open (XSI) requires to provide M_PI, but core POSIX does not */ #ifndef M_PI @@ -7124,6 +7125,29 @@ main(int argc, char **argv) setenv("PGDSQL", "1", 1); is_no_vacuum = true; foreign_keys = false; + + /* Initialize DSQL token generator */ + if (dsql_initialize_token_generator() != 0) + { + pg_fatal("Failed to initialize DSQL token generator"); + } + + /* Validate AWS credentials */ + { + char *err_msg = NULL; + if (dsql_validate_aws_credentials(&err_msg) != 0) + { + if (err_msg) + { + pg_fatal("DSQL credential validation failed: %s", err_msg); + free(err_msg); + } + else + { + pg_fatal("DSQL credential validation failed"); + } + } + } } /* set default script if none */ diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c index 291fd317b024f..abdcb57abf589 100644 --- a/src/bin/psql/startup.c +++ b/src/bin/psql/startup.c @@ -6,6 +6,7 @@ * src/bin/psql/startup.c */ #include "postgres_fe.h" +#include "libpq-fe.h" #ifndef WIN32 #include @@ -26,6 +27,8 @@ #include "mainloop.h" #include "settings.h" +#include "fe-dsql-auth.h" + /* * Global psql options */ @@ -221,6 +224,29 @@ main(int argc, char *argv[]) { setenv("PGDSQL", "1", 1); pset.getPassword = TRI_NO; + + /* Initialize DSQL token generator */ + if (dsql_initialize_token_generator() != 0) + { + pg_fatal("Failed to initialize DSQL token generator"); + } + + /* Validate AWS credentials */ + { + char *err_msg = NULL; + if (dsql_validate_aws_credentials(&err_msg) != 0) + { + if (err_msg) + { + pg_fatal("DSQL credential validation failed: %s", err_msg); + free(err_msg); + } + else + { + pg_fatal("DSQL credential validation failed"); + } + } + } } /* diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index 03556fcd12c49..883aaa78da118 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -90,15 +90,19 @@ ifeq ($(PORTNAME), linux) # Link with AWS libraries using start-group to resolve dependencies $(LD) -r -o fe-dsql-auth-with-aws.o fe-dsql-auth-temp.o --start-group $(AWS_DSQL_AUTH_ALL_LIBS) --end-group # Create a list of symbols to keep (only the public API from fe-dsql-auth.h) - echo "generate_dsql_token" > keep-symbols.txt - echo "dsql_auth_cleanup" >> keep-symbols.txt + echo "dsql_initialize_token_generator" > keep-symbols.txt + echo "dsql_generate_token" >> keep-symbols.txt + echo "dsql_validate_aws_credentials" >> keep-symbols.txt + echo "dsql_cleanup" >> keep-symbols.txt # Hide all symbols except the ones we want to keep objcopy --keep-global-symbols=keep-symbols.txt fe-dsql-auth-with-aws.o $@ rm -f fe-dsql-auth-temp.o fe-dsql-auth-with-aws.o keep-symbols.txt else ifeq ($(PORTNAME), darwin) # macOS: Use ld with exported symbols list - echo "_generate_dsql_token" > exported-symbols.txt - echo "_dsql_auth_cleanup" >> exported-symbols.txt + echo "_dsql_initialize_token_generator" > exported-symbols.txt + echo "_dsql_generate_token" >> exported-symbols.txt + echo "_dsql_validate_aws_credentials" >> exported-symbols.txt + echo "_dsql_cleanup" >> exported-symbols.txt $(LD) -r -o $@ fe-dsql-auth-temp.o -exported_symbols_list exported-symbols.txt $(AWS_DSQL_AUTH_ALL_LIBS) rm -f fe-dsql-auth-temp.o exported-symbols.txt else diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index 0625cf39e9af3..7ccf0b80cbd03 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -211,3 +211,7 @@ PQgetAuthDataHook 208 PQdefaultAuthDataHook 209 PQfullProtocolVersion 210 appendPQExpBufferVA 211 +dsql_initialize_token_generator 212 +dsql_generate_token 213 +dsql_validate_aws_credentials 214 +dsql_cleanup 215 diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 682b3b064c406..55cd08889ca0e 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -1441,7 +1441,7 @@ pqConnectOptions2(PGconn *conn) pwhost = conn->connhost[i].hostaddr; is_admin = strcmp("admin", conn->pguser) == 0; - token = generate_dsql_token(pwhost, is_admin, &err_msg); + token = dsql_generate_token(pwhost, is_admin, &err_msg); if (!token) { libpq_append_conn_error(conn, "DSQL token generation failed for host=%s: %s", diff --git a/src/interfaces/libpq/fe-dsql-auth.c b/src/interfaces/libpq/fe-dsql-auth.c index c7ed75d300eaf..8fa042d1b31e7 100644 --- a/src/interfaces/libpq/fe-dsql-auth.c +++ b/src/interfaces/libpq/fe-dsql-auth.c @@ -18,6 +18,8 @@ /* Include AWS DSQL Auth library functions */ #include #include +#include +#include #include #include #include @@ -37,6 +39,17 @@ static struct aws_event_loop_group *s_el_group = NULL; static struct aws_host_resolver *s_host_resolver = NULL; static struct aws_client_bootstrap *s_client_bootstrap = NULL; +/* + * DSQL Token Generator - holds state for efficient token generation + */ +struct dsql_token_generator { + struct aws_allocator *allocator; + struct aws_credentials_provider *credentials_provider; +}; + +/* Global token generator instance */ +static struct dsql_token_generator s_token_generator = {0}; + /* * Initialize DSQL logging */ @@ -47,7 +60,7 @@ initialize_dsql_logging(void) { struct aws_allocator *allocator = aws_default_allocator(); struct aws_logger_standard_options logger_options = { - .level = AWS_LOG_LEVEL_DEBUG, /* Can be controlled by environment variable */ + .level = AWS_LOG_LEVEL_NONE, /* Can be controlled by environment variable */ .file = stderr /* Log to stderr by default */ }; @@ -90,29 +103,6 @@ initialize_dsql_logging(void) } } -/* - * Clean up DSQL authentication resources - */ -void -dsql_auth_cleanup(void) -{ - if (dsql_logger_initialized) - { - aws_logger_set(NULL); - aws_logger_clean_up(&dsql_logger); - dsql_logger_initialized = false; - } - - if (aws_libs_initialized) - { - aws_sdkutils_library_clean_up(); - aws_auth_library_clean_up(); - aws_io_library_clean_up(); - aws_common_library_clean_up(); - aws_libs_initialized = false; - } -} - /* * Initialize AWS libraries if not already initialized */ @@ -121,7 +111,11 @@ initialize_aws_libs(void) { if (!aws_libs_initialized) { - struct aws_allocator *allocator = aws_default_allocator(); + struct aws_allocator *allocator; + struct aws_host_resolver_default_options resolver_options; + struct aws_client_bootstrap_options bootstrap_options; + + allocator = aws_default_allocator(); aws_common_library_init(allocator); aws_io_library_init(allocator); aws_http_library_init(allocator); @@ -137,20 +131,16 @@ initialize_aws_libs(void) goto error; } - struct aws_host_resolver_default_options resolver_options = { - .el_group = s_el_group, - .max_entries = 8, - }; + resolver_options.el_group = s_el_group; + resolver_options.max_entries = 8; s_host_resolver = aws_host_resolver_new_default(allocator, &resolver_options); if (!s_host_resolver) { AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create host resolver"); goto error; } - struct aws_client_bootstrap_options bootstrap_options = { - .event_loop_group = s_el_group, - .host_resolver = s_host_resolver, - }; + bootstrap_options.event_loop_group = s_el_group; + bootstrap_options.host_resolver = s_host_resolver; s_client_bootstrap = aws_client_bootstrap_new(allocator, &bootstrap_options); if (!s_client_bootstrap) { AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create client bootstrap"); @@ -183,41 +173,130 @@ initialize_aws_libs(void) } } +/* + * Initialize the DSQL token generator with long-lived components + */ +static int +initialize_token_generator(void) +{ + struct aws_credentials_provider_chain_default_options credentials_options; + + if (s_token_generator.allocator != NULL) { + /* Already initialized */ + return AWS_OP_SUCCESS; + } + + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Initializing DSQL token generator"); + + s_token_generator.allocator = aws_default_allocator(); + + /* Create credentials provider with client bootstrap for IMDS */ + AWS_ZERO_STRUCT(credentials_options); + credentials_options.bootstrap = s_client_bootstrap; + + s_token_generator.credentials_provider = aws_credentials_provider_new_chain_default( + s_token_generator.allocator, &credentials_options); + + if (!s_token_generator.credentials_provider) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create credentials provider for token generator"); + s_token_generator.allocator = NULL; + return AWS_OP_ERR; + } + + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "DSQL token generator initialized successfully"); + return AWS_OP_SUCCESS; +} + +/* + * Public initialization function for the DSQL token generator + */ +int +dsql_initialize_token_generator(void) +{ + /* Initialize AWS libraries and logging */ + initialize_aws_libs(); + initialize_dsql_logging(); + + return initialize_token_generator(); +} + +/* + * Clean up the DSQL token generator + */ +static void +cleanup_token_generator(void) +{ + if (s_token_generator.allocator != NULL) { + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Cleaning up DSQL token generator"); + + if (s_token_generator.credentials_provider) { + aws_credentials_provider_release(s_token_generator.credentials_provider); + s_token_generator.credentials_provider = NULL; + } + + s_token_generator.allocator = NULL; + } +} + +/* + * Clean up DSQL authentication resources + */ +void +dsql_cleanup(void) +{ + cleanup_token_generator(); + + if (dsql_logger_initialized) + { + aws_logger_set(NULL); + aws_logger_clean_up(&dsql_logger); + dsql_logger_initialized = false; + } + + if (aws_libs_initialized) + { + aws_sdkutils_library_clean_up(); + aws_auth_library_clean_up(); + aws_io_library_clean_up(); + aws_common_library_clean_up(); + aws_libs_initialized = false; + } +} + /* * Generate a DSQL authentication token for the specified endpoint. - * Uses the AWS DSQL auth library to generate a real token. + * Uses a local auth_config for thread safety and cached credentials provider for efficiency. * Returns a newly allocated string containing the token. */ char * -generate_dsql_token(const char *endpoint, bool admin, char **err_msg) +dsql_generate_token(const char *endpoint, bool admin, char **err_msg) { - struct aws_allocator *allocator; - struct aws_dsql_auth_config auth_config; + struct aws_dsql_auth_config auth_config = {0}; struct aws_dsql_auth_token auth_token = {0}; struct aws_string *aws_region = NULL; - struct aws_credentials_provider *credentials_provider = NULL; - struct aws_credentials_provider_chain_default_options credentials_options; char *token = NULL; int aws_error; const char *env_region; const char *token_str; - /* Initialize AWS libraries and logging */ - initialize_aws_libs(); - initialize_dsql_logging(); - - allocator = aws_default_allocator(); + /* Check if token generator is initialized */ + if (s_token_generator.allocator == NULL) { + if (err_msg) + *err_msg = strdup("Token generator not initialized"); + return NULL; + } AWS_LOGF_INFO(AWS_LS_AUTH_GENERAL, "Starting DSQL token generation for endpoint: %s", endpoint); - /* Initialize DSQL auth config */ + /* Initialize a local auth config for thread safety */ if (aws_dsql_auth_config_init(&auth_config) != AWS_OP_SUCCESS) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to initialize local auth config"); if (err_msg) - *err_msg = strdup("Failed to initialize DSQL auth config"); - goto cleanup; + *err_msg = strdup("Failed to initialize auth config"); + return NULL; } - /* Set hostname */ + /* Set hostname on the local auth config */ aws_dsql_auth_config_set_hostname(&auth_config, endpoint); /* Try to get region from environment variable first */ @@ -225,7 +304,7 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) if (env_region != NULL && env_region[0] != '\0') { AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Using AWS_REGION from environment: %s", env_region); - aws_region = aws_string_new_from_c_str(allocator, env_region); + aws_region = aws_string_new_from_c_str(s_token_generator.allocator, env_region); if (!aws_region) { AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to create region string from AWS_REGION"); if (err_msg) @@ -237,7 +316,7 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) { AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "AWS_REGION not set, attempting to infer from hostname: %s", endpoint); /* Try to infer region from hostname */ - if (aws_dsql_auth_config_infer_region(allocator, &auth_config, &aws_region) != AWS_OP_SUCCESS || + if (aws_dsql_auth_config_infer_region(s_token_generator.allocator, &auth_config, &aws_region) != AWS_OP_SUCCESS || aws_region == NULL) { AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to infer AWS region from hostname: %s", endpoint); @@ -249,28 +328,15 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) } aws_dsql_auth_config_set_region(&auth_config, aws_region); - /* Create default credentials provider with client bootstrap for IMDS */ - AWS_ZERO_STRUCT(credentials_options); - credentials_options.bootstrap = s_client_bootstrap; - - AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Creating credentials provider chain with bootstrap for IMDS"); - credentials_provider = aws_credentials_provider_new_chain_default(allocator, &credentials_options); - if (!credentials_provider) { - aws_error = aws_last_error(); - if (err_msg) - *err_msg = strdup(aws_error_str(aws_error)); - goto cleanup; - } - - /* Set credentials provider */ - aws_dsql_auth_config_set_credentials_provider(&auth_config, credentials_provider); + /* Set the cached credentials provider */ + aws_dsql_auth_config_set_credentials_provider(&auth_config, s_token_generator.credentials_provider); /* Set expiration time to 5 seconds for shorter token lifetime */ aws_dsql_auth_config_set_expires_in(&auth_config, 5); /* 5 seconds */ - /* Generate the token */ + /* Generate the token using local auth config and cached components */ AWS_ZERO_STRUCT(auth_token); - if (aws_dsql_auth_token_generate(&auth_config, admin, allocator, &auth_token) != AWS_OP_SUCCESS) + if (aws_dsql_auth_token_generate(&auth_config, admin, s_token_generator.allocator, &auth_token) != AWS_OP_SUCCESS) { aws_error = aws_last_error(); if (err_msg) @@ -283,10 +349,11 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) if (token_str) { token = strdup(token_str); - /* Token generation successful */ + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "DSQL token generated successfully using local auth config and cached credentials"); } else { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to get token string from generated token"); if (err_msg) *err_msg = strdup("Failed to get token string"); } @@ -295,11 +362,6 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) aws_dsql_auth_token_clean_up(&auth_token); aws_dsql_auth_config_clean_up(&auth_config); - if (credentials_provider) - { - aws_credentials_provider_release(credentials_provider); - } - if (aws_region) { aws_string_destroy(aws_region); @@ -307,3 +369,116 @@ generate_dsql_token(const char *endpoint, bool admin, char **err_msg) return token; } + +/* Synchronous credential retrieval state */ +struct credential_validation_state { + struct aws_credentials *credentials; + int error_code; + bool completed; + struct aws_mutex mutex; + struct aws_condition_variable condition_variable; +}; + +/* Callback for synchronous credential retrieval */ +static void +s_on_credentials_acquired(struct aws_credentials *credentials, int error_code, void *user_data) +{ + struct credential_validation_state *state = (struct credential_validation_state *)user_data; + + aws_mutex_lock(&state->mutex); + + state->credentials = credentials; + state->error_code = error_code; + state->completed = true; + + if (credentials) { + aws_credentials_acquire(credentials); + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Credentials acquired successfully for validation"); + } else { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Credentials acquisition failed with error: %d", error_code); + } + + aws_condition_variable_notify_one(&state->condition_variable); + aws_mutex_unlock(&state->mutex); +} + +/* + * Validate AWS credentials early for DSQL authentication. + * This initializes the token generator and validates that credentials can be obtained. + * Returns AWS_OP_SUCCESS on success, AWS_OP_ERR on failure. + */ +int +dsql_validate_aws_credentials(char **err_msg) +{ + struct credential_validation_state state = {0}; + int result = AWS_OP_ERR; + + /* Check if token generator is initialized */ + if (s_token_generator.allocator == NULL) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Token generator not initialized during credential validation"); + if (err_msg) + *err_msg = strdup("Token generator not initialized"); + return AWS_OP_ERR; + } + + AWS_LOGF_INFO(AWS_LS_AUTH_GENERAL, "Validating AWS credentials for DSQL authentication"); + + /* Initialize synchronization primitives */ + if (aws_mutex_init(&state.mutex) != AWS_OP_SUCCESS) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to initialize mutex for credential validation"); + if (err_msg) + *err_msg = strdup("Failed to initialize synchronization"); + return AWS_OP_ERR; + } + + if (aws_condition_variable_init(&state.condition_variable) != AWS_OP_SUCCESS) { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to initialize condition variable for credential validation"); + aws_mutex_clean_up(&state.mutex); + if (err_msg) + *err_msg = strdup("Failed to initialize synchronization"); + return AWS_OP_ERR; + } + + /* Actually retrieve credentials to validate they exist and are accessible */ + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Attempting to retrieve AWS credentials for validation"); + + if (aws_credentials_provider_get_credentials( + s_token_generator.credentials_provider, + s_on_credentials_acquired, + &state) != AWS_OP_SUCCESS) { + + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to initiate credentials retrieval"); + if (err_msg) + *err_msg = strdup("Failed to initiate credentials retrieval"); + goto cleanup; + } + + /* Wait for credentials retrieval to complete */ + aws_mutex_lock(&state.mutex); + while (!state.completed) { + aws_condition_variable_wait(&state.condition_variable, &state.mutex); + } + aws_mutex_unlock(&state.mutex); + + /* Check if credentials were successfully retrieved */ + if (state.credentials && state.error_code == AWS_OP_SUCCESS) { + AWS_LOGF_INFO(AWS_LS_AUTH_GENERAL, "AWS credentials validation completed successfully"); + AWS_LOGF_DEBUG(AWS_LS_AUTH_GENERAL, "Token generator ready for DSQL authentication"); + result = AWS_OP_SUCCESS; + + aws_credentials_release(state.credentials); + } else { + AWS_LOGF_ERROR(AWS_LS_AUTH_GENERAL, "Failed to retrieve AWS credentials: %s", + aws_error_str(state.error_code)); + if (err_msg) { + const char *error_str = aws_error_str(state.error_code); + *err_msg = strdup(error_str ? error_str : "Unknown credential retrieval error"); + } + } + +cleanup: + aws_condition_variable_clean_up(&state.condition_variable); + aws_mutex_clean_up(&state.mutex); + + return result; +} diff --git a/src/interfaces/libpq/fe-dsql-auth.h b/src/interfaces/libpq/fe-dsql-auth.h index 23c65c2f0cd5a..d24c3a600c7d7 100644 --- a/src/interfaces/libpq/fe-dsql-auth.h +++ b/src/interfaces/libpq/fe-dsql-auth.h @@ -10,10 +10,16 @@ #include +/* Initialize the DSQL token generator */ +int dsql_initialize_token_generator(void); + /* Generate a DSQL authentication token for the specified endpoint */ -char *generate_dsql_token(const char *endpoint, bool admin, char **err_msg); +char *dsql_generate_token(const char *endpoint, bool admin, char **err_msg); + +/* Initialize and validate AWS credentials early (for startup validation) */ +int dsql_validate_aws_credentials(char **err_msg); /* Clean up DSQL authentication resources */ -void dsql_auth_cleanup(void); +void dsql_cleanup(void); #endif /* FE_DSQL_AUTH_H */ From 996390bcb56101767831b11fc68db0dac03537fd Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 22:39:31 -0700 Subject: [PATCH 40/42] Fix missing zeros --- src/interfaces/libpq/fe-dsql-auth.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/interfaces/libpq/fe-dsql-auth.c b/src/interfaces/libpq/fe-dsql-auth.c index 8fa042d1b31e7..6163302a38cab 100644 --- a/src/interfaces/libpq/fe-dsql-auth.c +++ b/src/interfaces/libpq/fe-dsql-auth.c @@ -131,6 +131,7 @@ initialize_aws_libs(void) goto error; } + AWS_ZERO_STRUCT(resolver_options); resolver_options.el_group = s_el_group; resolver_options.max_entries = 8; s_host_resolver = aws_host_resolver_new_default(allocator, &resolver_options); @@ -139,6 +140,7 @@ initialize_aws_libs(void) goto error; } + AWS_ZERO_STRUCT(bootstrap_options); bootstrap_options.event_loop_group = s_el_group; bootstrap_options.host_resolver = s_host_resolver; s_client_bootstrap = aws_client_bootstrap_new(allocator, &bootstrap_options); From d67c60c5b242cb9458b99fa2560a7f15fa019bf9 Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Fri, 27 Jun 2025 22:57:33 -0700 Subject: [PATCH 41/42] Support private endpoints, install pgbench --- aws-dsql-auth | 2 +- scripts/install.sh | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/aws-dsql-auth b/aws-dsql-auth index f33fce18f3864..3c53ec315ed61 160000 --- a/aws-dsql-auth +++ b/aws-dsql-auth @@ -1 +1 @@ -Subproject commit f33fce18f3864e9428b10ae63c018b004b18436a +Subproject commit 3c53ec315ed619715c56875e81476119c365ddac diff --git a/scripts/install.sh b/scripts/install.sh index 7959634c7d33b..0dcbba2d306d3 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -196,11 +196,14 @@ install_via_zip() { # Clean up temp files rm -rf "$TEMP_DIR" - # Make the binary executable + # Make the binaries executable chmod +x "$INSTALL_PATH/bin/pdsql" + chmod +x "$INSTALL_PATH/bin/pgbench" echo "ZIP installation completed successfully!" - echo "PostgreSQL DSQL (pdsql) installed to: $INSTALL_PATH/bin/pdsql" + echo "PostgreSQL DSQL tools installed to:" + echo " - pdsql (DSQL client): $INSTALL_PATH/bin/pdsql" + echo " - pgbench (benchmark tool): $INSTALL_PATH/bin/pgbench" # Check if installation path is in PATH if [[ ":$PATH:" != *":$INSTALL_PATH/bin:"* ]]; then From 9607202c8d2b68c1c4ee80523f4ad23e6ead438f Mon Sep 17 00:00:00 2001 From: Marc Bowes <15209+marcbowes@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:32:09 -0700 Subject: [PATCH 42/42] History pk --- src/bin/pgbench/pgbench.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 967612a185828..49adc6dfcc1fe 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -4859,8 +4859,8 @@ initCreateTables(PGconn *con) static const struct ddlinfo DDLs[] = { { "pgbench_history", - "id uuid default gen_random_uuid(),tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)", - "id uuid default gen_random_uuid(),tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)", + "id uuid primary key default gen_random_uuid(),tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)", + "id uuid primary key default gen_random_uuid(),tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)", 0 }, {