8000 chore: add pg_egress_collect support to AIO image · openmodels-base/postgres@422a00b · GitHub
[go: up one dir, main page]
Skip to content

Commit 422a00b

Browse files
pcncdarora
pcnc
authored and
darora
committed
chore: add pg_egress_collect support to AIO image
1 parent b3e089c commit 422a00b

File tree

4 files changed

+149
-0
lines changed

4 files changed

+149
-0
lines changed

‎docker/all-in-one/Dockerfile

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
136136
sudo \
137137
# reading files using cat sucks
138138
less \
139+
# pg_egress_collect deps
140+
tcpdump libio-async-perl \
139141
&& rm -rf /var/lib/apt/lists/* /tmp/*
140142

141143
# Copy single binary dependencies
@@ -152,6 +154,9 @@ COPY ansible/files/admin_api_scripts /root
152154
COPY --chown=adminapi:adminapi docker/all-in-one/etc/adminapi /etc/adminapi
153155
COPY docker/all-in-one/etc/sudoers.d /etc/sudoers.d/
154156

157+
# Script for pg_egress_collect
158+
COPY --chown=adminapi:adminapi docker/all-in-one/opt/pg_egress_collect /opt/pg_egress_collect
159+
155160
# Customizations for pgbouncer
156161
COPY docker/all-in-one/etc/pgbouncer /etc/pgbouncer
157162
COPY docker/all-in-one/etc/pgbouncer-custom /etc/pgbouncer-custom

‎docker/all-in-one/etc/pg_egress_collect/pg_egress_collect.pl

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/usr/bin/env perl
2+
3+
# This script receive tcpdump output through STDIN and does:
4+
#
5+
# 1. extract outgoing TCP packet length on the 1st non-loopback device port 5432 and 6543
6+
# 2. sum the length up to one minute
7+
# 3. save the total length to file (default is /tmp/pg_egress_collect.txt) per minute
8+
#
9+
# Usage:
10+
#
11+
# tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl pg_egress_collect.pl -o /tmp/output.txt
12+
#
13+
14+
use POSIX;
15+
use List::Util qw(sum);
16+
use Getopt::Long 'HelpMessage';
17+
use IO::Async::Loop;
18+
use IO::Async::Stream;
19+
use IO::Async::Timer::Periodic;
20+
21+
use strict;
22+
use warnings;
23+
24+
# total captured packets lenth in a time frame
25+
my $captured_len = 0;
26+
27+
# extract tcp packet length captured by tcpdump
28+
#
29+
# Sample input lines:
30+
#
31+
# 167401383 10000 3.940253 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60)
32+
# 10.112.101.122.5432 > 220.235.16.223.62599: Flags [S.], cksum 0x5de3 (incorrect -> 0x63da), seq 2314200657, ack 2071735457, win 62643, options [mss 8961,sackOK,TS val 3358598837 ecr 1277499190,nop,wscale 7], length 0
33+
# 1674013833.989257 IP (tos 0x0, ttl 64, id 24975, offset 0, flags [DF], proto TCP (6), length 52)
34+
# 10.112.101.122.5432 > 220.235.16.223.62599: Flags [.], cksum 0x5ddb (incorrect -> 0xa25b), seq 1, ack 9, win 490, options [nop,nop,TS val 3358598885 ecr 1277499232], length 0
35+
sub extract_packet_length {
36+
my ($line) = @_;
37+
38+
#print("debug: >> " . $line);
39+
40+
if ($line =~ /^\s+\d+\.\d+\.\d+\.\d+\..*, length (\d+)$/) {
41+
# extract tcp packet length and add it up
42+
my $len = $1;
43+
$captured_len += $len;
44+
}
45+
}
46+
47+
# write total length to file
48+
sub write_file {
49+
my ($output) = @_;
50+
51+
my $now = strftime "%F %T", localtime time;
52+
print "[$now] write captured len $captured_len to $output\n";
53+
54+
open(my $fh, "+>", $output) or die "Could not open file '$output' $!";
55+
print $fh "$captured_len";
56+
close($fh) or die "Could not write file '$output' $!";
57+
}
58+
59+
# main
60+
sub main {
61+
# get arguments
62+
GetOptions(
63+
"interval:i" => \(my $interval = 60),
64+
"output:s" => \(my $output = "/tmp/pg_egress_collect.txt"),
65+
"help" => sub { HelpMessage(0) },
66+
) or HelpMessage(1);
67+
68+
my $loop = IO::Async::Loop->new;
69+
70+
# tcpdump extractor
71+
my $extractor = IO::Async::Stream->new_for_stdin(
72+
on_read => sub {
73+
my ($self, $buffref, $eof) = @_;
74+
75+
while($$buffref =~ s/^(.*\n)//) {
76+
my $line = $1;
77+
extract_packet_length($line);
78+
}
79+
80+
return 0;
81+
},
82+
);
83+
84+
# schedule file writer per minute
85+
my $writer = IO::Async::Timer::Periodic->new(
86+
interval => $interval,
87+
on_tick => sub {
88+
write_file($output);
89+
90+
# reset total captured length
91+
$captured_len = 0;
92+
},
93+
);
94+
$writer->start;
95+
96+
print "pg_egress_collect started, egress data will be saved to $output at interval $interval seconds.\n";
97+
98+
$loop->add($extractor);
99+
$loop->add($writer);
100+
$loop->run;
101+
}
102+
103+
main();
104+
105+
__END__
106+
107+
=head1 NAME
108+
109+
pg_egress_collect.pl - collect egress from tcpdump output, extract TCP packet length, aggregate in specified interval and write to output file.
110+
111+
=head1 SYNOPSIS
112+
113+
pg_egress_collect.pl [-i interval] [-o output]
114+
115+
Options:
116+
117+
-i, --interval interval
118+
output file write interval, in seconds, default is 60 seconds
119+
120+
-o, --output output
121+
output file path, default is /tmp/pg_egress_collect.txt
122+
123+
-h, --help
124+
print this help message
125+
126+
=cut

‎docker/all-in-one/etc/supervisor/db-only/pg_egress_collect.conf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[program:pg_egress_collect]
2+
command=/bin/bash -c "tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl /opt/pg_egress_collect/pg_egress_collect.pl"
3+
user=root
4+
autorestart=true
5+
autostart=true
6+
stdout_logfile=/var/log/services/pg_egress_collect.log
7+
redirect_stderr=true
8+
stdout_logfile_maxbytes=10MB
9+
priority=50

‎docker/all-in-one/init/configure-pg_egress_collect.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
set -eou pipefail
3+
4+
PG_EGRESS_COLLECT_FILE=/tmp/pg_egress_collect.txt
5+
6+
if [ "${DATA_VOLUME_MOUNTPOINT:-}" != "" ]; then
7+
touch "${DATA_VOLUME_MOUNTPOINT}/pg_egress_collect.txt"
8+
ln -s "${DATA_VOLUME_MOUNTPOINT}/pg_egress_collect.txt" $PG_EGRESS_COLLECT_FILE
9+
fi

0 commit comments

Comments
 (0)
0