From ebed3233d6291eca03ce8a3f3fc2ed9bc130165a Mon Sep 17 00:00:00 2001 From: Bruno Andrade <1145012+bmpandrade@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:01:11 +0000 Subject: [PATCH 1/3] Revert "chore: remove pgbouncer from docker aio image (#907)" This reverts commit d0bca07f068faabcadb8ad29dc16738cc59718ec. --- Dockerfile | 1 + common.vars.pkr.hcl | 2 +- docker/all-in-one/Dockerfile | 40 +- docker/all-in-one/entrypoint.sh | 20 +- docker/all-in-one/etc/adminapi/adminapi.yaml | 3 + .../etc/fail2ban/filter.d/postgresql.conf | 1 + .../etc/fail2ban/jail.d/pgbouncer.conf | 7 + docker/all-in-one/etc/pgbouncer/pgbouncer.ini | 363 ++++++++++++++++++ docker/all-in-one/etc/pgbouncer/userlist.txt | 0 docker/all-in-one/etc/postgresql.schema.sql | 2 + docker/all-in-one/etc/sudoers.d/adminapi | 2 + .../base-services/pg_egress_collect.conf | 2 +- .../etc/supervisor/services/pgbouncer.conf | 10 + .../all-in-one/etc/tmpfiles.d/pgbouncer.conf | 2 + docker/all-in-one/etc/vector/vector.yaml | 49 +++ docker/all-in-one/healthcheck.sh | 15 +- docker/all-in-one/init/configure-adminapi.sh | 2 + docker/all-in-one/init/configure-pgbouncer.sh | 46 +++ docker/all-in-one/init/configure-vector.sh | 3 + .../pg_egress_collect/pg_egress_collect.pl | 4 +- docker/orioledb/Dockerfile | 1 + 21 files changed, 547 insertions(+), 28 deletions(-) create mode 100644 docker/all-in-one/etc/fail2ban/jail.d/pgbouncer.conf create mode 100644 docker/all-in-one/etc/pgbouncer/pgbouncer.ini create mode 100644 docker/all-in-one/etc/pgbouncer/userlist.txt create mode 100644 docker/all-in-one/etc/supervisor/services/pgbouncer.conf create mode 100644 docker/all-in-one/etc/tmpfiles.d/pgbouncer.conf create mode 100755 docker/all-in-one/init/configure-pgbouncer.sh diff --git a/Dockerfile b/Dockerfile index ed7be1f31..a1e41e87a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -927,6 +927,7 @@ RUN sed -i \ # Include schema migrations COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql # Add upstream entrypoint script diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index f08904266..68b5f7068 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.26" +postgres-version = "15.1.1.25" diff --git a/docker/all-in-one/Dockerfile b/docker/all-in-one/Dockerfile index a29a1adbf..5e144a661 100644 --- a/docker/all-in-one/Dockerfile +++ b/docker/all-in-one/Dockerfile @@ -1,5 +1,6 @@ ARG postgres_version=15.1.1.24 +ARG pgbouncer_release=1.18.0 ARG postgrest_release=10.1.2 ARG gotrue_release=2.130.0 ARG adminapi_release=0.62.0 @@ -25,6 +26,33 @@ RUN apt-get update && apt-get install -y \ cmake \ && rm -rf /var/lib/apt/lists/* +#################### +# Install pgbouncer +#################### +FROM builder as pgbouncer-source +# Download and extract +ARG pgbouncer_release +ADD "https://www.pgbouncer.org/downloads/files/${pgbouncer_release}/pgbouncer-${pgbouncer_release}.tar.gz" /tmp/pgbouncer.tar.gz +RUN tar -xvf /tmp/pgbouncer.tar.gz -C /tmp && \ + rm -rf /tmp/pgbouncer.tar.gz +# Install build dependencies +RUN apt-get update && apt-get install -y \ + libevent-dev \ + && rm -rf /var/lib/apt/lists/* +# Build from source +WORKDIR /tmp/pgbouncer-${pgbouncer_release} +RUN ./configure --prefix=/usr/local +RUN make -j$(nproc) +# Create debian package +RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --requires=libevent-2.1-7 --nodoc + +FROM base as pgbouncer +# Download pre-built packages +RUN apt-get update && apt-get install -y --no-install-recommends --download-only \ + pgbouncer \ + && rm -rf /var/lib/apt/lists/* +RUN mv /var/cache/apt/archives/*.deb /tmp/ + #################### # Install PostgREST #################### @@ -98,6 +126,7 @@ RUN mv /var/cache/apt/archives/*.deb /tmp/ FROM base as production # Copy dependencies from previous build stages +COPY --from=pgbouncer /tmp/*.deb /tmp/ COPY --from=vector /tmp/*.deb /tmp/ COPY --from=kong /tmp/*.deb /tmp/ COPY --from=supervisor /tmp/*.deb /tmp/ @@ -148,6 +177,11 @@ COPY docker/all-in-one/etc/sudoers.d /etc/sudoers.d/ # Script for pg_egress_collect COPY --chown=adminapi:adminapi docker/all-in-one/opt/pg_egress_collect /opt/pg_egress_collect +# Customizations for pgbouncer +COPY docker/all-in-one/etc/pgbouncer /etc/pgbouncer +COPY docker/all-in-one/etc/pgbouncer-custom /etc/pgbouncer-custom +COPY docker/all-in-one/etc/tmpfiles.d /etc/tmpfiles.d + # Customizations for postgres COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql/pg_hba.conf /etc/postgresql/ COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql/logging.conf /etc/postgresql/ @@ -216,6 +250,9 @@ ENV ADMIN_API_CERT_DIR=/etc/ssl/adminapi ENV ADMIN_API_PORT=8085 EXPOSE ${ADMIN_API_PORT} +ENV PGBOUNCER_PORT=6543 +EXPOSE ${PGBOUNCER_PORT} + ENV PGEXPORTER_PORT=9187 EXPOSE ${PGEXPORTER_PORT} @@ -225,7 +262,8 @@ ENV VECTOR_API_PORT=9001 RUN useradd --create-home --shell /bin/bash postgrest && \ useradd --create-home --shell /bin/bash gotrue && \ useradd --create-home --shell /bin/bash envoy && \ - useradd --create-home --shell /bin/bash adminapi -G root,envoy,kong,postgres,postgrest,wal-g && \ + useradd --create-home --shell /bin/bash pgbouncer -G postgres,ssl-cert && \ + useradd --create-home --shell /bin/bash adminapi -G root,envoy,kong,pgbouncer,postgres,postgrest,wal-g && \ usermod --append --shell /bin/bash -G postgres vector RUN mkdir -p /etc/wal-g && \ chown -R adminapi:adminapi /etc/wal-g && \ diff --git a/docker/all-in-one/entrypoint.sh b/docker/all-in-one/entrypoint.sh index ed7fa1515..56b58cc39 100755 --- a/docker/all-in-one/entrypoint.sh +++ b/docker/all-in-one/entrypoint.sh @@ -84,17 +84,6 @@ function disable_fail2ban { sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/fail2ban.conf } -function disable_gotrue { - sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/gotrue.conf - sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/gotrue.conf -} - -function replace_kong_envoy { - sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/services/envoy.conf - sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/kong.conf - sed -i "s/kong/envoy/" /etc/supervisor/services/group.conf -} - function setup_postgres { tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql.schema.sql mv /etc/postgresql.schema.sql /docker-entrypoint-initdb.d/migrations/99-schema.sql @@ -285,18 +274,19 @@ if [ "${AUTOSHUTDOWN_ENABLED:-}" == "true" ]; then enable_autoshutdown fi - if [ "${ENVOY_ENABLED:-}" == "true" ]; then - replace_kong_envoy + sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/services/envoy.conf + sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/kong.conf + sed -i "s/kong/envoy/" /etc/supervisor/services/group.conf fi if [ "${FAIL2BAN_DISABLED:-}" == "true" ]; then disable_fail2ban fi - if [ "${GOTRUE_DISABLED:-}" == "true" ]; then - disable_gotrue + sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/gotrue.conf + sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/gotrue.conf fi if [ "${PLATFORM_DEPLOYMENT:-}" == "true" ]; then diff --git a/docker/all-in-one/etc/adminapi/adminapi.yaml b/docker/all-in-one/etc/adminapi/adminapi.yaml index 73be6473e..6eaf643e1 100644 --- a/docker/all-in-one/etc/adminapi/adminapi.yaml +++ b/docker/all-in-one/etc/adminapi/adminapi.yaml @@ -16,6 +16,8 @@ node_exporter_additional_args: # cert_path: /etc/ssl/adminapi/server.crt # key_path: /etc/ssl/adminapi/server.key upstream_metrics_refresh_duration: 60s +pgbouncer_endpoints: + - "postgres://pgbouncer:{{ .PgbouncerPassword }}@localhost:6543/pgbouncer" fail2ban_socket: /var/run/fail2ban/fail2ban.sock upstream_metrics_sources: - name: system @@ -60,6 +62,7 @@ firewall: - 0.0.0.0/0 filtered_ports: - 5432 + - 6543 unfiltered_ports: - 80 - 443 diff --git a/docker/all-in-one/etc/fail2ban/filter.d/postgresql.conf b/docker/all-in-one/etc/fail2ban/filter.d/postgresql.conf index 374db7a5a..4c708069d 100644 --- a/docker/all-in-one/etc/fail2ban/filter.d/postgresql.conf +++ b/docker/all-in-one/etc/fail2ban/filter.d/postgresql.conf @@ -5,3 +5,4 @@ ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for use ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_auth_admin".*$ ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_storage_admin".*$ ^.*,.*,.*,.*,":.*password authentication failed for user ""authenticator".*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""pgbouncer".*$ diff --git a/docker/all-in-one/etc/fail2ban/jail.d/pgbouncer.conf b/docker/all-in-one/etc/fail2ban/jail.d/pgbouncer.conf new file mode 100644 index 000000000..c8b3c49c5 --- /dev/null +++ b/docker/all-in-one/etc/fail2ban/jail.d/pgbouncer.conf @@ -0,0 +1,7 @@ +[pgbouncer] +enabled = true +port = 6543 +protocol = tcp +filter = pgbouncer +logpath = /var/log/services/pgbouncer.log +maxretry = 3 diff --git a/docker/all-in-one/etc/pgbouncer/pgbouncer.ini b/docker/all-in-one/etc/pgbouncer/pgbouncer.ini new file mode 100644 index 000000000..5a36ac197 --- /dev/null +++ b/docker/all-in-one/etc/pgbouncer/pgbouncer.ini @@ -0,0 +1,363 @@ +;;; +;;; PgBouncer configuration file +;;; + +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= auth_user= +;; client_encoding= datestyle= timezone= +;; pool_size= reserve_pool= max_db_connections= +;; pool_mode= connect_query= application_name= +[databases] +* = host=localhost auth_user=pgbouncer + +;; foodb over Unix socket +;foodb = + +;; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +;; access to dest database will go with single user +;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +;; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +;; use auth_user with auth_query if user not present in auth_file +;; auth_user must exist in auth_file +; foodb = auth_user=bar + +;; fallback connect string +;* = host=testserver + +;; User-specific configuration +[users] + +;user1 = pool_mode=transaction max_user_connections=10 + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +pidfile = /var/run/pgbouncer/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +;; IP address or * which means all IPs +listen_addr = * +listen_port = 6543 + +;; Unix socket is also used for -R. +;; On Debian it should be /var/run/postgresql +unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;client_tls_protocols = secure + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +;server_tls_sslmode = disable + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +;; any, trust, plain, md5, cert, hba, pam +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +auth_query = SELECT * FROM pgbouncer.get_auth($1) + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +;; comma-separated list of users who are allowed to change settings +admin_users = pgbouncer + +;; comma-separated list of users who are just allowed to use SHOW command +stats_users = pgbouncer + +;;; +;;; Pooler personality questions +;;; + +;; When server connection is released back to pool: +;; session - after client disconnects (default) +;; transaction - after transaction finishes +;; statement - after statement finishes +pool_mode = transaction + +;; Query for cleaning connection immediately after releasing from +;; client. No need to put ROLLBACK here, pgbouncer does not reuse +;; connections where transaction is left open. +;server_reset_query = DISCARD ALL + +;; Whether server_reset_query should run in all pooling modes. If it +;; is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +;; Comma-separated list of parameters to ignore when given in startup +;; packet. Newer JDBC versions require the extra_float_digits here. +ignore_startup_parameters = extra_float_digits + +;; When taking idle server into use, this query is run first. +;server_check_query = select 1 + +;; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +;; Close servers in session pooling mode after a RECONNECT, RELOAD, +;; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;; Period for updating aggregated stats. +;stats_period = 60 + +;;; +;;; Connection limits +;;; + +;; Total number of clients that can connect +;max_client_conn = 100 + +;; Default pool size. 20 is good number when transaction pooling +;; is in use, in session pooling it needs to be the number of +;; max clients you want to handle at any moment +default_pool_size = 15 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +;; If a clients needs to wait more than this many seconds, use reserve +;; pool. +;reserve_pool_timeout = 5 + +;; Maximum number of server connections for a database +;max_db_connections = 0 + +;; Maximum number of server connections for a user +;max_user_connections = 0 + +;; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +;; log if client connects or server connection is made +;log_connections = 1 + +;; log if and why connection was closed +;log_disconnections = 1 + +;; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; write aggregated stats into log +;log_stats = 1 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. Allows +;; to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return in +;; this time. Should be used to survive network problems, _not_ as +;; statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not +;; assigned to a server in this time. Should be used to limit the +;; number of queued queries in case of a database or network +;; failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this +;; time. Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they stay +;; unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; Close connections which are in "IDLE in transaction" state longer +;; than this many seconds. +;idle_transaction_timeout = 0 + +;; How long SUSPEND/-R waits for buffer flush before closing +;; connection. +;suspend_timeout = 10 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; Set SO_REUSEPORT socket option +;so_reuseport = 0 + +;; networking options, for info: man 7 tcp + +;; Linux: Notify program about new connection only if there is also +;; data received. (Seconds to wait.) On Linux the default is 45, on +;; other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. They also require +;; tcp_keepalive=1. + +;; Count of keepalive packets +;tcp_keepcnt = 0 + +;; How long the connection can be idle before sending keepalive +;; packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes +;tcp_keepintvl = 0 + +;; How long may transmitted data remain unacknowledged before TCP +;; connection is closed (in milliseconds) +;tcp_user_timeout = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;; Custom resolv.conf file, to set custom DNS servers or other options +;; (default: empty = use OS settings) +;resolv_conf = /etc/pgbouncer/resolv.conf + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL injection: when PQexec +;; is disabled, multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP +;; By default contains config file from command line. +;conffile + +;; Windows service name to register as. job_name is alias for +;; service_name, used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from other file +;%include /etc/pgbouncer/pgbouncer-other.ini + +%include /etc/pgbouncer-custom/generated-optimizations.ini +%include /etc/pgbouncer-custom/custom-overrides.ini +# %include /etc/pgbouncer-custom/ssl-config.ini diff --git a/docker/all-in-one/etc/pgbouncer/userlist.txt b/docker/all-in-one/etc/pgbouncer/userlist.txt new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one/etc/postgresql.schema.sql b/docker/all-in-one/etc/postgresql.schema.sql index 2b4a0c018..475b09bb9 100644 --- a/docker/all-in-one/etc/postgresql.schema.sql +++ b/docker/all-in-one/etc/postgresql.schema.sql @@ -1,5 +1,6 @@ \set admin_pass `echo "${SUPABASE_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` \set pgrst_pass `echo "${AUTHENTICATOR_PASSWORD:-$POSTGRES_PASSWORD}"` +\set pgbouncer_pass `echo "${PGBOUNCER_PASSWORD:-$POSTGRES_PASSWORD}"` \set auth_pass `echo "${SUPABASE_AUTH_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` \set storage_pass `echo "${SUPABASE_STORAGE_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` \set replication_pass `echo "${SUPABASE_REPLICATION_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` @@ -7,6 +8,7 @@ ALTER USER supabase_admin WITH PASSWORD :'admin_pass'; ALTER USER authenticator WITH PASSWORD :'pgrst_pass'; +ALTER USER pgbouncer WITH PASSWORD :'pgbouncer_pass'; ALTER USER supabase_auth_admin WITH PASSWORD :'auth_pass'; ALTER USER supabase_storage_admin WITH PASSWORD :'storage_pass'; ALTER USER supabase_replication_admin WITH PASSWORD :'replication_pass'; diff --git a/docker/all-in-one/etc/sudoers.d/adminapi b/docker/all-in-one/etc/sudoers.d/adminapi index 620b9cf94..eeff5fb9c 100644 --- a/docker/all-in-one/etc/sudoers.d/adminapi +++ b/docker/all-in-one/etc/sudoers.d/adminapi @@ -2,6 +2,7 @@ Cmnd_Alias ENVOY = /usr/bin/supervisorctl start services\:envoy, /usr/bin/superv Cmnd_Alias KONG = /usr/bin/supervisorctl start services\:kong, /usr/bin/supervisorctl stop services\:kong, /usr/bin/supervisorctl restart services\:kong, /usr/bin/supervisorctl status services\:kong Cmnd_Alias POSTGREST = /usr/bin/supervisorctl start services\:postgrest, /usr/bin/supervisorctl stop services\:postgrest, /usr/bin/supervisorctl restart services\:postgrest, /usr/bin/supervisorctl status services\:postgrest Cmnd_Alias GOTRUE = /usr/bin/supervisorctl start services\:gotrue, /usr/bin/supervisorctl stop services\:gotrue, /usr/bin/supervisorctl restart services\:gotrue, /usr/bin/supervisorctl status services\:gotrue +Cmnd_Alias PGBOUNCER = /usr/bin/supervisorctl start pgbouncer, /usr/bin/supervisorctl stop pgbouncer, /usr/bin/supervisorctl restart pgbouncer, /usr/bin/supervisorctl status pgbouncer %adminapi ALL= NOPASSWD: /root/grow_fs.sh %adminapi ALL= NOPASSWD: /root/manage_readonly_mode.sh @@ -23,3 +24,4 @@ Cmnd_Alias GOTRUE = /usr/bin/supervisorctl start services\:gotrue, /usr/bin/supe %adminapi ALL= NOPASSWD: KONG %adminapi ALL= NOPASSWD: POSTGREST %adminapi ALL= NOPASSWD: GOTRUE +%adminapi ALL= NOPASSWD: PGBOUNCER diff --git a/docker/all-in-one/etc/supervisor/base-services/pg_egress_collect.conf b/docker/all-in-one/etc/supervisor/base-services/pg_egress_collect.conf index 41cf3fc08..de166beab 100644 --- a/docker/all-in-one/etc/supervisor/base-services/pg_egress_collect.conf +++ b/docker/all-in-one/etc/supervisor/base-services/pg_egress_collect.conf @@ -1,5 +1,5 @@ [program:pg_egress_collect] -command=/bin/bash -c "tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and port 5432' | perl /opt/pg_egress_collect/pg_egress_collect.pl" +command=/bin/bash -c "tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl /opt/pg_egress_collect/pg_egress_collect.pl" user=root autorestart=true autostart=true diff --git a/docker/all-in-one/etc/supervisor/services/pgbouncer.conf b/docker/all-in-one/etc/supervisor/services/pgbouncer.conf new file mode 100644 index 000000000..44fe93b5f --- /dev/null +++ b/docker/all-in-one/etc/supervisor/services/pgbouncer.conf @@ -0,0 +1,10 @@ +[program:pgbouncer] +command=/usr/sbin/pgbouncer /etc/pgbouncer/pgbouncer.ini +user=pgbouncer +stopsignal=INT +autorestart=true +autostart=true +stdout_logfile=/var/log/services/pgbouncer.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=150 diff --git a/docker/all-in-one/etc/tmpfiles.d/pgbouncer.conf b/docker/all-in-one/etc/tmpfiles.d/pgbouncer.conf new file mode 100644 index 000000000..d5d2cd49d --- /dev/null +++ b/docker/all-in-one/etc/tmpfiles.d/pgbouncer.conf @@ -0,0 +1,2 @@ +# Directory for PostgreSQL sockets, lockfiles and stats tempfiles +d /run/pgbouncer 2775 pgbouncer postgres - - \ No newline at end of file diff --git a/docker/all-in-one/etc/vector/vector.yaml b/docker/all-in-one/etc/vector/vector.yaml index b6ced43f2..0fdc99473 100644 --- a/docker/all-in-one/etc/vector/vector.yaml +++ b/docker/all-in-one/etc/vector/vector.yaml @@ -10,6 +10,11 @@ sources: include: - /var/log/services/postgrest.log + pgbouncer_log: + type: file + include: + - /var/log/services/pgbouncer.log + postgres_log: type: file include: @@ -89,6 +94,13 @@ transforms: condition: >- contains!(.message, "password authentication failed for user") + filter_pgbouncer_stats: + type: filter + inputs: + - pgbouncer_log + condition: >- + !starts_with!(.message, "stats:") && !starts_with!(.message, "kernel file descriptor limit") && !contains!(.message, "FIXME") + filter_postgrest_stats: type: filter inputs: @@ -168,6 +180,34 @@ transforms: del(.__MONOTONIC_TIMESTAMP) del(.__REALTIME_TIMESTAMP) + pgbouncer_to_object: + inputs: + - filter_pgbouncer_stats + type: remap + source: |2- + .project = "{{ .ProjectRef }}" + .metadata.host = del(.host) + del(.source_type) + del(.PRIORITY) + del(.SYSLOG_IDENTIFIER) + del(._BOOT_ID) + del(._CAP_EFFECTIVE) + del(._CMDLINE) + del(._COMM) + del(._EXE) + del(._GID) + del(._MACHINE_ID) + del(._PID) + del(._SELINUX_CONTEXT) + del(._SOURCE_REALTIME_TIMESTAMP) + del(._SYSTEMD_CGROUP) + del(._SYSTEMD_INVOCATION_ID) + del(._SYSTEMD_SLICE) + del(._SYSTEMD_UNIT) + del(._TRANSPORT) + del(._UID) + del(.__MONOTONIC_TIMESTAMP) + del(.__REALTIME_TIMESTAMP) sinks: http_gotrue: type: "http" @@ -193,6 +233,15 @@ sinks: retry_max_duration_secs: 10 uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .PostgrestSource }}" + http_pgbouncer: + type: http + inputs: + - pgbouncer_to_object + encoding: + codec: json + compression: none + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .PgbouncerSource }}" + http_postgres: type: http inputs: diff --git a/docker/all-in-one/healthcheck.sh b/docker/all-in-one/healthcheck.sh index d03aaab3c..039b461f1 100755 --- a/docker/all-in-one/healthcheck.sh +++ b/docker/all-in-one/healthcheck.sh @@ -22,10 +22,8 @@ fi # postgrest up curl -sSfI "http://localhost:$PGRST_ADMIN_SERVER_PORT/ready" -if [ "${GOTRUE_DISABLED:-}" != "true" ]; then - # gotrue up - curl -sSf "http://localhost:$GOTRUE_API_PORT/health" -fi +# gotrue up +curl -sSf "http://localhost:$GOTRUE_API_PORT/health" if [ "${ENVOY_ENABLED:-}" == "true" ]; then # envoy up @@ -35,10 +33,11 @@ else kong health fi -if [ "${FAIL2BAN_DISABLED:-}" != "true" ]; then - # fail2ban up - fail2ban-client status -fi +# pgbouncer up +printf \\0 > "/dev/tcp/localhost/$PGBOUNCER_PORT" + +# fail2ban up +fail2ban-client status # prometheus exporter up curl -sSfI "http://localhost:$PGEXPORTER_PORT/metrics" diff --git a/docker/all-in-one/init/configure-adminapi.sh b/docker/all-in-one/init/configure-adminapi.sh index 2f2f9691f..f88c09160 100755 --- a/docker/all-in-one/init/configure-adminapi.sh +++ b/docker/all-in-one/init/configure-adminapi.sh @@ -20,10 +20,12 @@ if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then chmod 700 -R $ADMIN_API_CERT_DIR else PROJECT_REF=${PROJECT_REF:-default} + PGBOUNCER_PASSWORD=${PGBOUNCER_PASSWORD:-$POSTGRES_PASSWORD} SUPABASE_URL=${SUPABASE_URL:-https://api.supabase.io/system} REPORTING_TOKEN=${REPORTING_TOKEN:-token} sed -i "s|{{ .JwtSecret }}|$JWT_SECRET|g" $ADMIN_API_CONF + sed -i "s|{{ .PgbouncerPassword }}|$PGBOUNCER_PASSWORD|g" $ADMIN_API_CONF sed -i "s|{{ .ProjectRef }}|$PROJECT_REF|g" $ADMIN_API_CONF sed -i "s|{{ .SupabaseUrl }}|$SUPABASE_URL|g" $ADMIN_API_CONF sed -i "s|{{ .ReportingToken }}|$REPORTING_TOKEN|g" $ADMIN_API_CONF diff --git a/docker/all-in-one/init/configure-pgbouncer.sh b/docker/all-in-one/init/configure-pgbouncer.sh new file mode 100755 index 000000000..ea30b9a2c --- /dev/null +++ b/docker/all-in-one/init/configure-pgbouncer.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -eou pipefail + +touch /var/log/services/pgbouncer.log + +mkdir -p /var/run/pgbouncer +chown pgbouncer:postgres /var/run/pgbouncer + +PGBOUNCER_CONF=/etc/pgbouncer/pgbouncer.ini + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init pgbouncer payload" + sed -i -E "s|^# (%include /etc/pgbouncer-custom/ssl-config.ini)$|\1|g" $PGBOUNCER_CONF + + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/pgbouncer/ --strip-components 3 ./etc/pgbouncer/userlist.txt + chown -R pgbouncer:pgbouncer /etc/pgbouncer/userlist.txt +fi + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + /opt/supabase-admin-api optimize pgbouncer --destination-config-file-path /etc/pgbouncer-custom/generated-optimizations.ini + + # Preserve pgbouncer configs across restarts + PGBOUNCER_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/pgbouncer" + PGBOUNCER_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/pgbouncer-custom" + + mkdir -p "${PGBOUNCER_DIR}" + mkdir -p "${PGBOUNCER_CUSTOM_DIR}" + + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom pgbouncer config from /etc/pgbouncer-custom to ${PGBOUNCER_CUSTOM_DIR}" + cp -R "/etc/pgbouncer-custom/." "${PGBOUNCER_CUSTOM_DIR}/" + cp -R "/etc/pgbouncer/." "${PGBOUNCER_DIR}/" + fi + + rm -rf "/etc/pgbouncer-custom" + ln -s "${PGBOUNCER_CUSTOM_DIR}" "/etc/pgbouncer-custom" + chown -R pgbouncer:pgbouncer "/etc/pgbouncer-custom" + chown -R pgbouncer:pgbouncer "${PGBOUNCER_CUSTOM_DIR}" + chmod -R g+rx "${PGBOUNCER_CUSTOM_DIR}" + + rm -rf "/etc/pgbouncer" + ln -s "${PGBOUNCER_DIR}" "/etc/pgbouncer" + chown -R pgbouncer:pgbouncer "/etc/pgbouncer" + chown -R pgbouncer:pgbouncer "${PGBOUNCER_DIR}" + chmod -R g+wrx "${PGBOUNCER_DIR}" +fi diff --git a/docker/all-in-one/init/configure-vector.sh b/docker/all-in-one/init/configure-vector.sh index 3e7ca3743..94415ef69 100755 --- a/docker/all-in-one/init/configure-vector.sh +++ b/docker/all-in-one/init/configure-vector.sh @@ -11,6 +11,7 @@ if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then LOGFLARE_DB_SOURCE=$(jq -r '.["logflare_db_source"]' /etc/vector/init.json) LOGFLARE_GOTRUE_SOURCE=$(jq -r '.["logflare_gotrue_source"]' /etc/vector/init.json) LOGFLARE_POSTGREST_SOURCE=$(jq -r '.["logflare_postgrest_source"]' /etc/vector/init.json) + LOGFLARE_PGBOUNCER_SOURCE=$(jq -r '.["logflare_pgbouncer_source"]' /etc/vector/init.json) LOGFLARE_API_KEY=$(jq -r '.["logflare_api_key"]' /etc/vector/init.json) fi @@ -40,6 +41,7 @@ LOGFLARE_HOST=${LOGFLARE_HOST:-api.logflare.app} LOGFLARE_DB_SOURCE=${LOGFLARE_DB_SOURCE:-postgres.logs} LOGFLARE_GOTRUE_SOURCE=${LOGFLARE_GOTRUE_SOURCE:-gotrue.logs.prod} LOGFLARE_POSTGREST_SOURCE=${LOGFLARE_POSTGREST_SOURCE:-postgREST.logs.prod} +LOGFLARE_PGBOUNCER_SOURCE=${LOGFLARE_PGBOUNCER_SOURCE:-pgbouncer.logs.prod} sed -i "s|{{ .ApiPort }}|$VECTOR_API_PORT|g" $VECTOR_CONF sed -i "s|{{ .ProjectRef }}|$PROJECT_REF|g" $VECTOR_CONF @@ -48,3 +50,4 @@ sed -i "s|{{ .ApiKey }}|$LOGFLARE_API_KEY|g" $VECTOR_CONF sed -i "s|{{ .DbSource }}|$LOGFLARE_DB_SOURCE|g" $VECTOR_CONF sed -i "s|{{ .GotrueSource }}|$LOGFLARE_GOTRUE_SOURCE|g" $VECTOR_CONF sed -i "s|{{ .PostgrestSource }}|$LOGFLARE_POSTGREST_SOURCE|g" $VECTOR_CONF +sed -i "s|{{ .PgbouncerSource }}|$LOGFLARE_PGBOUNCER_SOURCE|g" $VECTOR_CONF diff --git a/docker/all-in-one/opt/pg_egress_collect/pg_egress_collect.pl b/docker/all-in-one/opt/pg_egress_collect/pg_egress_collect.pl index 7948b0b00..2acc98aa6 100644 --- a/docker/all-in-one/opt/pg_egress_collect/pg_egress_collect.pl +++ b/docker/all-in-one/opt/pg_egress_collect/pg_egress_collect.pl @@ -2,13 +2,13 @@ # This script receive tcpdump output through STDIN and does: # -# 1. extract outgoing TCP packet length on the 1st non-loopback device port 5432 +# 1. extract outgoing TCP packet length on the 1st non-loopback device port 5432 and 6543 # 2. sum the length up to one minute # 3. save the total length to file (default is /tmp/pg_egress_collect.txt) per minute # # Usage: # -# tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and port 5432' | perl pg_egress_collect.pl -o /tmp/output.txt +# tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl pg_egress_collect.pl -o /tmp/output.txt # use POSIX; diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index db7716042..494ffff4c 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -1025,6 +1025,7 @@ RUN sed -i \ # Include schema migrations COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql # Patch upstream entrypoint script From 25a20b446337f752d6d2184ae3485d5fbd83ee3c Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Wed, 20 Mar 2024 17:03:34 +0000 Subject: [PATCH 2/3] fix: bump version to 15.1.1.27 as .26 was disabled due to Fly issues with docker aio --- common.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 68b5f7068..a75995eb6 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.25" +postgres-version = "15.1.1.27" From 4c6ffaa622db46e7749f735fe58dd6ec8f760b5a Mon Sep 17 00:00:00 2001 From: Div Arora Date: Thu, 21 Mar 2024 14:21:27 +0700 Subject: [PATCH 3/3] Update common.vars.pkr.hcl --- common.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index a75995eb6..1f267be6a 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.27" +postgres-version = "15.1.1.28"