diff --git a/Dockerfile-pgapt b/Dockerfile-pgapt index 06bfd96..bf64f61 100644 --- a/Dockerfile-pgapt +++ b/Dockerfile-pgapt @@ -1,4 +1,5 @@ FROM ubuntu +ARG FEATURES ARG PG_VERSION ARG PHASE1_ACTION ARG PHASE2_ACTION @@ -19,6 +20,8 @@ RUN p=$EXTRA_OS_PACKAGES; p="${p%\"}";p="${p#\"}"; \ git gcc make libreadline-dev zlib1g-dev libicu-dev bison flex gettext \ openjdk-17-jdk maven sysbench +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then apt install -y linux-tools-common linux-tools-`uname -r`; fi + RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 @@ -76,7 +79,12 @@ printf "\n"\ "host all all 0.0.0.0/0 md5\n"\ "host all all ::1/128 trust\n" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf -RUN echo "/usr/lib/postgresql/$PG_VERSION/bin/postgres -D /var/lib/postgresql/$PG_VERSION/main -c config_file=/etc/postgresql/$PG_VERSION/main/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then \ + echo "perf record -o /home/postgres/results/perf.data --call-graph dwarf -- /usr/lib/postgresql/$PG_VERSION/bin/postgres -D /var/lib/postgresql/$PG_VERSION/main -c config_file=/etc/postgresql/$PG_VERSION/main/postgresql.conf >/home/postgres/results/perf.log 2>&1; sleep inf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + echo "/usr/lib/postgresql/$PG_VERSION/bin/pg_ctl -w -D /var/lib/postgresql/$PG_VERSION/main stop" >/usr/bin/pg_ctl_stop && chmod a+x /usr/bin/pg_ctl_stop; \ + else \ + echo "/usr/lib/postgresql/$PG_VERSION/bin/postgres -D /var/lib/postgresql/$PG_VERSION/main -c config_file=/etc/postgresql/$PG_VERSION/main/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + fi # Run the rest of the commands as the ``postgres`` user created by the ``postgres-$version`` package when it was ``apt installed`` USER postgres diff --git a/Dockerfile-proapt b/Dockerfile-proapt index 3a37114..867f867 100644 --- a/Dockerfile-proapt +++ b/Dockerfile-proapt @@ -1,4 +1,5 @@ FROM ubuntu +ARG FEATURES ARG PG_VERSION ARG PHASE1_ACTION ARG PHASE2_ACTION @@ -21,6 +22,8 @@ RUN p=$EXTRA_OS_PACKAGES; p="${p%\"}";p="${p#\"}"; \ git gcc make libreadline-dev zlib1g-dev libicu-dev bison flex gettext \ openjdk-17-jdk maven sysbench +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then apt install -y linux-tools-common linux-tools-`uname -r`; fi + RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 @@ -81,7 +84,12 @@ printf "\n"\ "host all all 0.0.0.0/0 md5\n"\ "host all all ::1/128 trust\n" >> /var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data/pg_hba.conf -RUN echo "/opt/pgpro/$PGPRO_EDN-$PG_VERSION/bin/postgres -D /var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data -c config_file=/var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then \ + echo "perf record -o /home/postgres/results/perf.data --call-graph dwarf -- /opt/pgpro/$PGPRO_EDN-$PG_VERSION/bin/postgres -D /var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data -c config_file=/var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data/postgresql.conf >/home/postgres/results/perf.log 2>&1; sleep inf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + echo "/opt/pgpro/$PGPRO_EDN-$PG_VERSION/bin/pg_ctl -w -D /var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data stop" >/usr/bin/pg_ctl_stop && chmod a+x /usr/bin/pg_ctl_stop; \ + else \ + echo "/opt/pgpro/$PGPRO_EDN-$PG_VERSION/bin/postgres -D /var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data -c config_file=/var/lib/pgpro/$PGPRO_EDN-$PG_VERSION/data/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + fi # Run the rest of the commands as the ``postgres`` user created by the ``postgres*-$version`` package when it was ``apt installed`` USER postgres diff --git a/Dockerfile-src b/Dockerfile-src index bd7a6da..1fdf5b5 100644 --- a/Dockerfile-src +++ b/Dockerfile-src @@ -1,4 +1,5 @@ FROM ubuntu +ARG FEATURES ARG PG_VERSION ARG PHASE1_ACTION ARG PHASE2_ACTION @@ -20,6 +21,8 @@ RUN p=$EXTRA_OS_PACKAGES; p="${p%\"}";p="${p#\"}"; \ git gcc make libreadline-dev zlib1g-dev libicu-dev bison flex gettext \ openjdk-17-jdk maven sysbench +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then apt install -y linux-tools-common linux-tools-`uname -r`; fi + RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.UTF-8 ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 @@ -58,7 +61,12 @@ RUN cd /home/postgres/src/postgres && make -s install && \ ENV PATH "$PATH:/usr/local/pgsql/bin" -RUN echo "/usr/local/pgsql/bin/postgres -D /var/lib/postgresql/main -c config_file=/var/lib/postgresql/main/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain +RUN if (echo "$FEATURES" | grep -Eq "\bperf\b"); then \ + echo "perf record -o /home/postgres/results/perf.data --call-graph dwarf -- /usr/local/pgsql/bin/postgres -D /var/lib/postgresql/main -c config_file=/var/lib/postgresql/main/postgresql.conf >/home/postgres/results/perf.log 2>&1; sleep inf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + echo "/usr/local/pgsql/bin/pg_ctl -w -D /var/lib/postgresql/main stop" >/usr/bin/pg_ctl_stop && chmod a+x /usr/bin/pg_ctl_stop; \ + else \ + echo "/usr/local/pgsql/bin/postgres -D /var/lib/postgresql/main -c config_file=/var/lib/postgresql/main/postgresql.conf" >/usr/bin/pgmain && chmod a+x /usr/bin/pgmain; \ + fi COPY ./$PHASE2_ACTION /bin/ RUN $PHASE2_ACTION diff --git a/config.xml b/config.xml index 58f921f..be13417 100644 --- a/config.xml +++ b/config.xml @@ -30,6 +30,7 @@ + diff --git a/prepare-instances.py b/prepare-instances.py index 5253559..c11beb3 100755 --- a/prepare-instances.py +++ b/prepare-instances.py @@ -142,12 +142,16 @@ def get_repo_url(instance): encoding='UTF-8') as dockerfile: dockerfile.write(df_contents) + if instance.get('features') is not None: + build_args += ["--build-arg", + (f'FEATURES={instance.get("features")}')] + if instance.get('pg_version') is not None: - build_args += ['--build-arg', ('PG_VERSION=' + - instance.get('pg_version'))] + build_args += ['--build-arg', + (f'PG_VERSION={instance.get("pg_version")}')] if instance.get('pgpro_edition') is not None: - build_args += ['--build-arg', ('PGPRO_EDN=' + - instance.get('pgpro_edition'))] + build_args += ['--build-arg', + (f'PGPRO_EDN={instance.get("pgpro_edition")}')] pg_params = '' for param in instance.findall('config/pg_param'): pg_params += f"{param.get('name')} = '{param.get('value')}'\\n" @@ -157,15 +161,15 @@ def get_repo_url(instance): repo_url = get_repo_url(instance) if repo_url is not None: - build_args += ['--build-arg', ('REPOSITORY=' + repo_url)] + build_args += ['--build-arg', (f'REPOSITORY={repo_url}')] extra = instance.find('extra') if extra is not None: if extra.find('source') is not None and \ extra.find('source').get('type') == 'git': extra_git_url = extra.find('source').get('url') - build_args += ['--build-arg', ('EXTRA_SRC_GIT=' + - extra_git_url)] + build_args += ['--build-arg', + (f'EXTRA_SRC_GIT={extra_git_url}')] pg_modules = '' for pgm in extra.findall('pg_module'): pg_modules += (',' if pg_modules else '') + pgm.get('name') @@ -182,8 +186,8 @@ def get_repo_url(instance): phase2_action = instance.find('os').get('phase2_action') if instance.find('os').get('extra_packages') is not None: extra_os_packages = instance.find('os').get('extra_packages') - build_args += ['--build-arg', ('PHASE1_ACTION=' + phase1_action)] - build_args += ['--build-arg', ('PHASE2_ACTION=' + phase2_action)] + build_args += ['--build-arg', (f'PHASE1_ACTION={phase1_action}')] + build_args += ['--build-arg', (f'PHASE2_ACTION={phase2_action}')] build_args += ['--build-arg', (f'EXTRA_OS_PACKAGES="{extra_os_packages}"')] diff --git a/run-benchmarks.py b/run-benchmarks.py index a9583e6..56c0b53 100755 --- a/run-benchmarks.py +++ b/run-benchmarks.py @@ -106,6 +106,8 @@ def main(configfile, instances, benchmarks, resultsfile, resultsdir): insts = {} if not instances: for inst in config.findall('./pg_instances//instance'): + if inst.get('disabled') == 'true': + continue insts[inst.get('id')] = inst else: inst_cnts = {} @@ -155,8 +157,6 @@ def main(configfile, instances, benchmarks, resultsfile, resultsdir): bmbench.set('id', bench_id) for instance_uid, instance in insts.items(): instance_id = instance.get('id') - if instance.get('disabled') == 'true': - continue if re.match(r'^[\./]', instance_id): raise Exception(f'Invalid image id: {instance_id}') @@ -188,7 +188,7 @@ def main(configfile, instances, benchmarks, resultsfile, resultsdir): f'-v {os.getcwd()}/resources:/home/postgres/resources ' f'-v {os.getcwd()}/scripts:/home/postgres/scripts ' f'-v {resultdir}:/home/postgres/results ' - f'--tty --cap-add=SYS_PTRACE ' + f'--tty --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN ' f'--shm-size=2g {envvars} {instance_id}', shell=True, check=True, stdout=subprocess.PIPE) container_id = res.stdout.decode('utf-8').strip() @@ -247,6 +247,28 @@ def main(configfile, instances, benchmarks, resultsfile, resultsdir): print(output) raise Exception('Benchmark execution failed! (For details ' 'see benchmark-results/.)') + + instance_features = instance.get('features') + if re.match(r'\bperf\b', instance_features): + res = run(f'docker exec -t {container_id} bash -c ' + f'pg_ctl_stop', + shell=True, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + print(res.stdout.decode('utf-8')) + raise Exception('Could not stop server.') + sleep(10) + res = run(f'docker exec -t {container_id} bash -c "' + f'perf report --stdio -i' + f' /home/postgres/results/perf.data' + f' >/home/postgres/results/perf.out 2>&1; ' + f'perf script -i' + f' /home/postgres/results/perf.data' + f' >/home/postgres/results/perf.script 2>&1"', + shell=True, check=False, stdout=subprocess.PIPE) + if res.returncode != 0: + print(res.stdout.decode('utf-8')) + raise Exception('Could not get perf report.') + for metric in bench.findall('./results/metric'): mid = metric.get('id') mre = metric.get('regexp')