8000 NodeApp is refactored by dmitry-lipetsk · Pull Request #278 · postgrespro/testgres · GitHub
[go: up one dir, main page]

Skip to content

NodeApp is refactored #278

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
17fb8c3
NodeApp was moved to own py-file (refactoring)
dmitry-lipetsk Jun 26, 2025
8eaa289
NodeApp is refactored totally
dmitry-lipetsk Jun 26, 2025
2246447
NodeApp::test_path is corrected
dmitry-lipetsk Jun 26, 2025
3fe3e94
Default value of 'pg_options' (NodeApp::make_simple) is None
dmitry-lipetsk Jun 27, 2025
c1afd7e
Merge branch 'master' into D20250626_001--node_app
dmitry-lipetsk Jun 27, 2025
1062a97
OsOperations::create_clone is added
dmitry-lipetsk Jun 27, 2025
1512fe7
[#277] NodeApp::make_empty does not allow None/empty base_dir
dmitry-lipetsk Jun 27, 2025
b5b0d2b
[#267] NodeApp.make_simple does not change 'initdb_params'
dmitry-lipetsk Jun 27, 2025
dffde2c
test_node_app__make_empty__base_dir_is_None is corrected
dmitry-lipetsk Jun 27, 2025
d5c3672
NodeApp::make_simple is corrected (bad assert)
dmitry-lipetsk Jun 27, 2025
aae53eb
test_node_app__make_empty__base_dir_is_xxx is updated
dmitry-lipetsk Jun 27, 2025
488d556
[#265] NodeApp.make_simple links a new node with own os_ops and port_…
dmitry-lipetsk Jun 27, 2025
688b89f
test_node_app__make_empty is corrected
dmitry-lipetsk Jun 27, 2025
4f85ce5
NodeApp::nodes_to_cleanup is RO-property
dmitry-lipetsk Jun 27, 2025
f79e9a7
flake8
dmitry-lipetsk Jun 27, 2025
5817f63
test_node_app__make_empty is fixed
dmitry-lipetsk Jun 28, 2025
0547b0b
PostgresNode::release_resources(self) is added
dmitry-lipetsk Jun 28, 2025
1b12ca3
[FIX] NodeApp::make_empty processes a failure of self._nodes_to_clean…
dmitry-lipetsk Jun 28, 2025
1af3dfa
New test test_node_app__make_empty_with_explicit_port is added
dmitry-lipetsk Jun 28, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
NodeApp was moved to own py-file (refactoring)
  • Loading branch information
dmitry-lipetsk committed Jun 26, 2025
commit 17fb8c334aca3372801858d01dfaf4a3b0c9729d
8 changes: 5 additions & 3 deletions testgres/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@
ProcessType, \
DumpFormat

from .node import PostgresNode, NodeApp
from .node import PostgresNode
from .node import PortManager
from .node_app import NodeApp

from .utils import \
reserve_port, \
Expand Down Expand Up @@ -62,8 +63,9 @@
"NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
"TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
"XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
"PostgresNode", "NodeApp",
"PortManager",
NodeApp.__name__,
PostgresNode.__name__,
PortManager.__name__,
"reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",
"First", "Any",
"OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"
Expand Down
163 changes: 0 additions & 163 deletions testgres/node.py
8000
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
import signal
import subprocess
import threading
import tempfile
import platform
from queue import Queue

import time
Expand Down Expand Up @@ -2352,164 +2350,3 @@ def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:
return True

return False


class NodeApp:

def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None):
assert os_ops is None or isinstance(os_ops, OsOperations)

if os_ops is None:
os_ops = LocalOperations.get_single_instance()

assert isinstance(os_ops, OsOperations)

if test_path:
if os.path.isabs(test_path):
self.test_path = test_path
else:
self.test_path = os_ops.build_path(os_ops.cwd(), test_path)
else:
self.test_path = os_ops.cwd()
self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else []
self.os_ops = os_ops

def make_empty(
self,
base_dir=None,
port=None,
bin_dir=None):
real_base_dir = self.os_ops.build_path(self.test_path, base_dir)
self.os_ops.rmdirs(real_base_dir, ignore_errors=True)
self.os_ops.makedirs(real_base_dir)

node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir)
self.nodes_to_cleanup.append(node)

return node

def make_simple(
self,
base_dir=None,
port=None,
set_replication=False,
ptrack_enable=False,
initdb_params=[],
pg_options={},
checksum=True,
bin_dir=None):
assert type(pg_options) == dict # noqa: E721

if checksum and '--data-checksums' not in initdb_params:
initdb_params.append('--data-checksums')
node = self.make_empty(base_dir, port, bin_dir=bin_dir)
node.init(
initdb_params=initdb_params, allow_streaming=set_replication)

# set major version
pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION'))
node.major_version_str = str(pg_version_file.rstrip())
node.major_version = float(node.major_version_str)

# Set default parameters
options = {
'max_connections': 100,
'shared_buffers': '10MB',
'fsync': 'off',
'wal_level': 'logical',
'hot_standby': 'off',
'log_line_prefix': '%t [%p]: [%l-1] ',
'log_statement': 'none',
'log_duration': 'on',
'log_min_duration_statement': 0,
'log_connections': 'on',
'log_disconnections': 'on',
'restart_after_crash': 'off',
'autovacuum': 'off',
# unix_socket_directories will be defined later
}

# Allow replication in pg_hba.conf
if set_replication:
options['max_wal_senders'] = 10

if ptrack_enable:
options['ptrack.map_size'] = '1'
options['shared_preload_libraries'] = 'ptrack'

if node.major_version >= 13:
options['wal_keep_size'] = '200MB'
else:
options['wal_keep_segments'] = '12'

# Apply given parameters
for option_name, option_value in iteritems(pg_options):
options[option_name] = option_value

# Define delayed propertyes
if not ("unix_socket_directories" in options.keys()):
options["unix_socket_directories"] = __class__._gettempdir_for_socket()

# Set config values
node.set_auto_conf(options)

# kludge for testgres
# https://github.com/postgrespro/testgres/issues/54
# for PG >= 13 remove 'wal_keep_segments' parameter
if node.major_version >= 13:
node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments'])

return node

@staticmethod
def _gettempdir_for_socket():
platform_system_name = platform.system().lower()

if platform_system_name == "windows":
return __class__._gettempdir()

#
# [2025-02-17] Hot fix.
#
# Let's use hard coded path as Postgres likes.
#
# pg_config_manual.h:
#
# #ifndef WIN32
# #define DEFAULT_PGSOCKET_DIR "/tmp"
# #else
# #define DEFAULT_PGSOCKET_DIR ""
# #endif
#
# On the altlinux-10 tempfile.gettempdir() may return
# the path to "private" temp directiry - "/temp/.private/<username>/"
#
# But Postgres want to find a socket file in "/tmp" (see above).
#

return "/tmp"

@staticmethod
def _gettempdir():
v = tempfile.gettempdir()

#
# Paranoid checks
#
if type(v) != str: # noqa: E721
__class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__))

if v == "":
__class__._raise_bugcheck("tempfile.gettempdir returned an empty string.")

if not os.path.exists(v):
__class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v))

# OK
return v

@staticmethod
def _raise_bugcheck(msg):
assert type(msg) == str # noqa: E721
assert msg != ""
raise Exception("[BUG CHECK] " + msg)
168 changes: 168 additions & 0 deletions testgres/node_app.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
from .node import OsOperations
from .node import LocalOperations
from .node import PostgresNode

import os
import platform
import tempfile


class NodeApp:

def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None):
assert os_ops is None or isinstance(os_ops, OsOperations)

if os_ops is None:
os_ops = LocalOperations.get_single_instance()

assert isinstance(os_ops, OsOperations)

if test_path:
if os.path.isabs(test_path):
self.test_path = test_path
else:
self.test_path = os_ops.build_path(os_ops.cwd(), test_path)
else:
self.test_path = os_ops.cwd()
self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else []
self.os_ops = os_ops

def make_empty(
self,
base_dir=None,
port=None,
bin_dir=None):
real_base_dir = self.os_ops.build_path(self.test_path, base_dir)
self.os_ops.rmdirs(real_base_dir, ignore_errors=True)
self.os_ops.makedirs(real_base_dir)

node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir)
self.nodes_to_cleanup.append(node)

return node

def make_simple(
self,
base_dir=None,
port=None,
set_replication=False,
ptrack_enable=False,
initdb_params=[],
pg_options={},
checksum=True,
bin_dir=None):
assert type(pg_options) == dict # noqa: E721

if checksum and '--data-checksums' not in initdb_params:
initdb_params.append('--data-checksums')
node = self.make_empty(base_dir, port, bin_dir=bin_dir)
node.init(
initdb_params=initdb_params, allow_streaming=set_replication)

# set major version
pg_version_file = self.os_ops.read(self.os_ops.build_path(node.data_dir, 'PG_VERSION'))
node.major_version_str = str(pg_version_file.rstrip())
node.major_version = float(node.major_version_str)

# Set default parameters
options = {
'max_connections': 100,
'shared_buffers': '10MB',
'fsync': 'off',
'wal_level': 'logical',
'hot_standby': 'off',
'log_line_prefix': '%t [%p]: [%l-1] ',
'log_statement': 'none',
'log_duration': 'on',
'log_min_duration_statement': 0,
'log_connections': 'on',
'log_disconnections': 'on',
'restart_after_crash': 'off',
'autovacuum': 'off',
# unix_socket_directories will be defined later
}

# Allow replication in pg_hba.conf
if set_replication:
options['max_wal_senders'] = 10

if ptrack_enable:
options['ptrack.map_size'] = '1'
options['shared_preload_libraries'] = 'ptrack'

if node.major_version >= 13:
options['wal_keep_size'] = '200MB'
else:
options['wal_keep_segments'] = '12'

# Apply given parameters
for option_name, option_value in pg_options.items():
options[option_name] = option_value

# Define delayed propertyes
if not ("unix_socket_directories" in options.keys()):
options["unix_socket_directories"] = __class__._gettempdir_for_socket()

# Set config values
node.set_auto_conf(options)

# kludge for testgres
# https://github.com/postgrespro/testgres/issues/54
# for PG >= 13 remove 'wal_keep_segments' parameter
if node.major_version >= 13:
node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments'])

return node

@staticmethod
def _gettempdir_for_socket():
platform_system_name = platform.system().lower()

if platform_system_name == "windows":
return __class__._gettempdir()

#
# [2025-02-17] Hot fix.
#
# Let's use hard coded path as Postgres likes.
#
# pg_config_manual.h:
#
# #ifndef WIN32
# #define DEFAULT_PGSOCKET_DIR "/tmp"
# #else
# #define DEFAULT_PGSOCKET_DIR ""
# #endif
#
# On the altlinux-10 tempfile.gettempdir() may return
# the path to "private" temp directiry - "/temp/.private/<username>/"
#
# But Postgres want to find a socket file in "/tmp" (see above).
#

return "/tmp"

@staticmethod
def _gettempdir():
v = tempfile.gettempdir()

#
# Paranoid checks
#
if type(v) != str: # noqa: E721
__class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__))

if v == "":
__class__._raise_bugcheck("tempfile.gettempdir returned an empty string.")

if not os.path.exists(v):
__class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v))

# OK
return v

@staticmethod
def _raise_bugcheck(msg):
assert type(msg) == str # noqa: E721
assert msg != ""
raise Exception("[BUG CHECK] " + msg)
0