From cb0a4a0fc1cd9b296b44248a30f6815e60d2d7cb Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Mon, 12 Oct 2026 18:02:21 +0700 Subject: [PATCH 01/11] Allow spawning nodes in a different execution context PostgresNode cloning currently always reuses the source node's OsOperations and PortManager. NodeBackup propagates that behavior to spawn_primary(), spawn_replica(), and PostgresNode.replicate(), which prevents a spawned node from running in another execution context. Add optional target_os_ops, target_host and target_port arguments and propagate them through the cloning and backup/replication path. When the target OsOperations differs, let PostgresNode select a PortManager for that context. Preserve the existing behavior when the new arguments are not specified. A physical backup contains the source node's listen_addresses setting, so override listen_addresses and port with the spawned node's values. Also remove the same-host assertion from source_walsender: master process inspection is already performed through the master's own OsOperations. Different source and target OsOperations are supported when both execution contexts can access the backup directory using the same path. --- src/backup.py | 59 +++++++++++++++++++++--- src/node.py | 53 +++++++++++++++++----- tests/test_testgres_common.py | 85 +++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 17 deletions(-) diff --git a/src/backup.py b/src/backup.py index 75ac1af2..f3a0ba41 100644 --- a/src/backup.py +++ b/src/backup.py @@ -139,13 +139,29 @@ def _prepare_dir(self, destroy): # Return path to new node return dest_base_dir - def spawn_primary(self, name=None, destroy=True): + def spawn_primary( + self, + name=None, + destroy=True, + target_os_ops=None, + target_host=None, + target_port=None, + ): """ Create a primary node from a backup. Args: name: primary's application name. destroy: should we convert this backup into a node? + target_os_ops: OS operations object for the new node. If omitted, + use the source node's os_ops. + target_host: host/address for the new node. If omitted, use the + source node's host. + target_port: explicit port for the new node. If omitted, reserve a + port in the target execution context. + + If target_os_ops differs from the source node's os_ops, both execution + contexts must see base_dir at the same path. Returns: New instance of :class:`.PostgresNode`. @@ -157,19 +173,38 @@ def spawn_primary(self, name=None, destroy=True): # Build a new PostgresNode assert self.original_node is not None - node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) + node = self.original_node.clone_with_new_name_and_base_dir( + name=name, + base_dir=base_dir, + target_os_ops=target_os_ops, + target_host=target_host, + target_port=target_port, + ) assert node is not None assert type(node) is self.original_node.__class__ with clean_on_error(node) as node: - # Set a new port + # The backup contains the source node's connection settings. + # Override settings that belong to the spawned node. node.append_conf(filename=PG_CONF_FILE, line='\n') - node.append_conf(filename=PG_CONF_FILE, port=node.port) + node.append_conf( + filename=PG_CONF_FILE, + listen_addresses=node.host, + port=node.port, + ) return node - def spawn_replica(self, name=None, destroy=True, slot=None): + def spawn_replica( + self, + name=None, + destroy=True, + slot=None, + target_os_ops=None, + target_host=None, + target_port=None, + ): """ Create a replica of the original node from a backup. @@ -177,13 +212,25 @@ def spawn_replica(self, name=None, destroy=True, slot=None): name: replica's application name. slot: create a replication slot with the specified name. destroy: should we convert this backup into a node? + target_os_ops: OS operations object for the replica. If omitted, + use the source node's os_ops. + target_host: host/address for the replica. If omitted, use the + source node's host. + target_port: explicit port for the replica. If omitted, reserve a + port in the target execution context. Returns: New instance of :class:`.PostgresNode`. """ # Build a new PostgresNode - node = self.spawn_primary(name=name, destroy=destroy) + node = self.spawn_primary( + name=name, + destroy=destroy, + target_os_ops=target_os_ops, + target_host=target_host, + target_port=target_port, + ) assert node is not None try: diff --git a/src/node.py b/src/node.py index 4706a287..f6749938 100644 --- a/src/node.py +++ b/src/node.py @@ -315,17 +315,35 @@ def _get_port_manager(os_ops: OsOperations) -> PortManager: # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic2(os_ops) - def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): + def clone_with_new_name_and_base_dir( + self, + name: str, + base_dir: str, + target_os_ops: typing.Optional[OsOperations] = None, + target_host: typing.Optional[str] = None, + target_port: typing.Optional[int] = None, + ): assert name is None or type(name) is str assert base_dir is None or type(base_dir) is str + assert target_os_ops is None or isinstance(target_os_ops, OsOperations) + assert target_host is None or type(target_host) is str + assert target_port is None or type(target_port) is int assert __class__ == PostgresNode - if self._port_manager is None: - raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + if target_os_ops is None: + target_os_ops = self._os_ops + + if target_port is None and target_os_ops is self._os_ops: + if self._port_manager is None: + raise InvalidOperationException( + "PostgresNode without PortManager can't be cloned." + ) + + target_port_manager = self._port_manager + else: + target_port_manager = None - assert self._port_manager is not None - assert isinstance(self._port_manager, PortManager) assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) assert type(self._host) is str @@ -333,11 +351,12 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): node = PostgresNode( name=name, base_dir=base_dir, + port=target_port, bin_dir=self._bin_dir, prefix=self._prefix, - os_ops=self._os_ops, - port_manager=self._port_manager, - host=self._host, + os_ops=target_os_ops, + port_manager=target_port_manager, + host=target_host if target_host is not None else self._host, ) return node @@ -535,9 +554,6 @@ def source_walsender(self): assert type(self.master) is PostgresNode - # master should be on the same host - assert self.master.host == self._host - with self.master.connect() as con: for row in con.execute(sql, self.name): for child in self.master.auxiliary_processes: @@ -2021,6 +2037,9 @@ def replicate( self, name=None, slot=None, + target_os_ops=None, + target_host=None, + target_port=None, **kwargs, ): """ @@ -2029,9 +2048,18 @@ def replicate( Args: name: replica's application name. slot: create a replication slot with the specified name. + target_os_ops: OS operations object for the replica. If omitted, + use this node's os_ops. + target_host: host/address for the replica. If omitted, use this + node's host. + target_port: explicit port for the replica. If omitted, reserve a + port in the target execution context. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs + + If target_os_ops differs from this node's os_ops, both execution + contexts must see the backup directory at the same path. """ # transform backup into a replica @@ -2040,6 +2068,9 @@ def replicate( name=name, destroy=True, slot=slot, + target_os_ops=target_os_ops, + target_host=target_host, + target_port=target_port, ) def set_synchronous_standbys(self, standbys): diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index d037e6a7..4f6259ac 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -17,6 +17,7 @@ from src.node import PostgresNodeLogReader from src.node import PostgresNodeUtils from src.node import ProcessProxy +from src.node import PortManager__Generic2 from src.utils import PgVer from src.utils import get_pg_version2 from src.utils import file_tail @@ -62,6 +63,7 @@ import psutil import threading import testgres.postgres_configuration as testgres_pgconf +from testgres.operations.local_ops import LocalOperations from packaging.version import Version from concurrent.futures import ThreadPoolExecutor @@ -239,6 +241,51 @@ def test_node_constructor__host(self): assert node2._port_manager is node._port_manager return + def test_node_clone__target_context(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + target_port = node_svc.port_manager.reserve_port() + try: + nodedir = node_svc.os_ops.mkdtemp(prefix="node-target--") + + with node.clone_with_new_name_and_base_dir( + name="target", + base_dir=nodedir, + target_os_ops=node_svc.os_ops, + target_host=node.host, + target_port=target_port, + ) as node2: + assert node2.os_ops is node_svc.os_ops + assert node2.host == node.host + assert node2.port == target_port + assert node2.port_manager is None + finally: + node_svc.port_manager.release_port(target_port) + + def test_node_clone__different_os_ops(self): + source_svc = PostgresNodeServices.sm_local + target_os_ops = LocalOperations() + + assert target_os_ops is not source_svc.os_ops + + unique_id = uuid.uuid4().hex + tmpdir = target_os_ops.get_tempdir() + + with __class__.helper__get_node(source_svc) as node: + nodedir = target_os_ops.build_path(tmpdir, "node-target--" + unique_id) + + with node.clone_with_new_name_and_base_dir( + name="target", + base_dir=nodedir, + target_os_ops=target_os_ops, + ) as node2: + assert node2.os_ops is target_os_ops + assert node2.host == node.host + assert node2.port != node.port + assert node2.port_manager is not node.port_manager + assert type(node2.port_manager) is PortManager__Generic2 + def test_node_repr(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -2051,6 +2098,44 @@ def test_replicate(self, node_svc: PostgresNodeService): res = node.execute('select * from test') assert (res == []) + def test_replicate__target_context(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as master: + master.init(allow_streaming=True).start() + + target_port = node_svc.port_manager.reserve_port() + try: + # For local operations use an equivalent host spelling to + # exercise source_walsender with different host strings. A + # remote host must keep its configured address. + if isinstance(node_svc.os_ops, LocalOperations): + target_host = "localhost" if master.host != "localhost" else "127.0.0.1" + else: + target_host = master.host + + with master.replicate( + target_os_ops=node_svc.os_ops, + target_host=target_host, + target_port=target_port, + ).start() as replica: + assert replica.os_ops is node_svc.os_ops + assert replica.host == target_host + assert replica.port == target_port + assert replica.port_manager is None + + master.execute('create table test_target_context (val int)', commit=True) + master.execute('insert into test_target_context values (42)', commit=True) + + replica.catchup() + + res = replica.execute('select val from test_target_context') + assert res == [(42, )] + + assert replica.source_walsender is not None + finally: + node_svc.port_manager.release_port(target_port) + def test_synchronous_replication(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) From bd5accb0da3fb1eab012561269a126ecb24b3db3 Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Tue, 13 Oct 2026 16:29:16 +0700 Subject: [PATCH 02/11] Run backups in the target execution context The target OsOperations was previously applied only when constructing the spawned PostgresNode. The backup itself was still created using the source node's OsOperations. This worked when source and target execution contexts shared a filesystem, such as Linux network namespaces, but did not support different filesystems or remote hosts. Allow NodeBackup to use an explicit OsOperations and make PostgresNode.replicate() create the backup directly in target_os_ops. This makes pg_basebackup, temporary directory creation, copytree and backup cleanup operate in the target execution context. Spawned nodes now inherit the backup's OsOperations. When cloning into a different OsOperations, do not inherit source bin_dir or host; let the target PostgresNode resolve them from its own execution context. Add tests for backup copying across execution contexts and physical replication from a remote source to a local target. --- src/backup.py | 40 +++++++++--------- src/node.py | 46 +++++++++++++++------ tests/test_testgres_common.py | 78 ++++++++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 34 deletions(-) diff --git a/src/backup.py b/src/backup.py index f3a0ba41..522dfb4a 100644 --- a/src/backup.py +++ b/src/backup.py @@ -35,7 +35,8 @@ def __init__(self, base_dir=None, username=None, xlog_method=XLogMethod.fetch, - options=None): + options=None, + os_ops=None): """ Create a new backup. @@ -44,13 +45,18 @@ def __init__(self, base_dir: where should we store it? username: database user name. xlog_method: none | fetch | stream (see docs) + os_ops: execution context where pg_basebackup is run and the + backup is stored. If omitted, use the source node's os_ops. + The source node must be reachable from this context using its + configured host and port. """ assert node.os_ops is not None assert isinstance(node.os_ops, OsOperations) + assert os_ops is None or isinstance(os_ops, OsOperations) if not options: options = [] - self.os_ops = node.os_ops + self.os_ops = os_ops if os_ops is not None else node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -63,7 +69,9 @@ def __init__(self, raise BackupException(msg) # Set default arguments - username = username or self.os_ops.get_user() + # The username belongs to the source PostgreSQL instance, not to the + # environment where pg_basebackup happens to run. + username = username or node.os_ops.get_user() base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP) # public @@ -143,7 +151,6 @@ def spawn_primary( self, name=None, destroy=True, - target_os_ops=None, target_host=None, target_port=None, ): @@ -153,15 +160,11 @@ def spawn_primary( Args: name: primary's application name. destroy: should we convert this backup into a node? - target_os_ops: OS operations object for the new node. If omitted, - use the source node's os_ops. - target_host: host/address for the new node. If omitted, use the - source node's host. + target_host: host/address for the new node. If omitted, preserve + the source host when the backup uses the source os_ops, + otherwise use the backup os_ops host. target_port: explicit port for the new node. If omitted, reserve a - port in the target execution context. - - If target_os_ops differs from the source node's os_ops, both execution - contexts must see base_dir at the same path. + port in the backup's execution context. Returns: New instance of :class:`.PostgresNode`. @@ -176,7 +179,7 @@ def spawn_primary( node = self.original_node.clone_with_new_name_and_base_dir( name=name, base_dir=base_dir, - target_os_ops=target_os_ops, + target_os_ops=self.os_ops, target_host=target_host, target_port=target_port, ) @@ -201,7 +204,6 @@ def spawn_replica( name=None, destroy=True, slot=None, - target_os_ops=None, target_host=None, target_port=None, ): @@ -212,12 +214,11 @@ def spawn_replica( name: replica's application name. slot: create a replication slot with the specified name. destroy: should we convert this backup into a node? - target_os_ops: OS operations object for the replica. If omitted, - use the source node's os_ops. - target_host: host/address for the replica. If omitted, use the - source node's host. + target_host: host/address for the replica. If omitted, preserve + the source host when the backup uses the source os_ops, + otherwise use the backup os_ops host. target_port: explicit port for the replica. If omitted, reserve a - port in the target execution context. + port in the backup's execution context. Returns: New instance of :class:`.PostgresNode`. @@ -227,7 +228,6 @@ def spawn_replica( node = self.spawn_primary( name=name, destroy=destroy, - target_os_ops=target_os_ops, target_host=target_host, target_port=target_port, ) diff --git a/src/node.py b/src/node.py index f6749938..9c14686f 100644 --- a/src/node.py +++ b/src/node.py @@ -334,7 +334,9 @@ def clone_with_new_name_and_base_dir( if target_os_ops is None: target_os_ops = self._os_ops - if target_port is None and target_os_ops is self._os_ops: + same_os_ops = target_os_ops is self._os_ops + + if target_port is None and same_os_ops: if self._port_manager is None: raise InvalidOperationException( "PostgresNode without PortManager can't be cloned." @@ -344,6 +346,14 @@ def clone_with_new_name_and_base_dir( else: target_port_manager = None + # Paths and default host names belong to an OsOperations context. + # Reusing them for another context is only safe when it is the same + # OsOperations object. Otherwise let PostgresNode resolve them using + # the target environment. + target_bin_dir = self._bin_dir if same_os_ops else None + if target_host is None and same_os_ops: + target_host = self._host + assert self._os_ops is not None assert isinstance(self._os_ops, OsOperations) assert type(self._host) is str @@ -352,11 +362,11 @@ def clone_with_new_name_and_base_dir( name=name, base_dir=base_dir, port=target_port, - bin_dir=self._bin_dir, + bin_dir=target_bin_dir, prefix=self._prefix, os_ops=target_os_ops, port_manager=target_port_manager, - host=target_host if target_host is not None else self._host, + host=target_host, ) return node @@ -2023,6 +2033,10 @@ def backup( username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs + os_ops: execution context where pg_basebackup is run and the + backup is stored. If omitted, use this node's os_ops. This + node must be reachable from that context using its configured + host and port. Returns: A smart object of type NodeBackup. @@ -2049,26 +2063,34 @@ def replicate( name: replica's application name. slot: create a replication slot with the specified name. target_os_ops: OS operations object for the replica. If omitted, - use this node's os_ops. - target_host: host/address for the replica. If omitted, use this - node's host. + use this node's os_ops. pg_basebackup is executed in this + context and writes the backup directly there. The primary must + be reachable from this context using its configured host and + port. + target_host: host/address for the replica. If omitted, preserve + the source host in the same execution context, otherwise use + the host from target_os_ops. target_port: explicit port for the replica. If omitted, reserve a port in the target execution context. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs - - If target_os_ops differs from this node's os_ops, both execution - contexts must see the backup directory at the same path. """ - # transform backup into a replica - with utils.clean_on_error(self.backup(**kwargs)) as backup: + backup_os_ops = ( + target_os_ops if target_os_ops is not None else self._os_ops + ) + + # Create the backup in the same execution context where the replica + # will live. This avoids copying a backup across OsOperations + # instances and also makes _prepare_dir() operate in the right context. + with utils.clean_on_error( + self.backup(os_ops=backup_os_ops, **kwargs) + ) as backup: return backup.spawn_replica( name=name, destroy=True, slot=slot, - target_os_ops=target_os_ops, target_host=target_host, target_port=target_port, ) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4f6259ac..abe83bb9 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -273,7 +273,10 @@ def test_node_clone__different_os_ops(self): tmpdir = target_os_ops.get_tempdir() with __class__.helper__get_node(source_svc) as node: - nodedir = target_os_ops.build_path(tmpdir, "node-target--" + unique_id) + nodedir = target_os_ops.build_path( + tmpdir, + "node-target--" + unique_id, + ) with node.clone_with_new_name_and_base_dir( name="target", @@ -281,7 +284,7 @@ def test_node_clone__different_os_ops(self): target_os_ops=target_os_ops, ) as node2: assert node2.os_ops is target_os_ops - assert node2.host == node.host + assert node2.host == target_os_ops.host assert node2.port != node.port assert node2.port_manager is not node.port_manager assert type(node2.port_manager) is PortManager__Generic2 @@ -1913,6 +1916,35 @@ def test_backup_multiple(self, node_svc: PostgresNodeService): backup.spawn_primary('node2', destroy=False) as node2: assert (node1.base_dir != node2.base_dir) + @pytest.mark.parametrize( + ("source_svc", "target_svc"), + [ + pytest.param( + PostgresNodeServices.sm_remote, + PostgresNodeServices.sm_local, + id="remote_to_local", + ), + ], + ) + def test_backup__different_os_ops( + self, + source_svc: PostgresNodeService, + target_svc: PostgresNodeService, + ): + assert source_svc.os_ops is not target_svc.os_ops + + with __class__.helper__get_node(source_svc) as source: + source.init(allow_streaming=True).start() + + with source.backup(os_ops=target_svc.os_ops) as backup: + assert backup.os_ops is target_svc.os_ops + + with backup.spawn_primary(destroy=False) as target: + assert target.os_ops is target_svc.os_ops + assert target.host == target_svc.os_ops.host + assert target.base_dir != backup.base_dir + assert target_svc.os_ops.isdir(target.data_dir) + def test_backup_exhaust(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc) as node: @@ -2136,6 +2168,48 @@ def test_replicate__target_context(self, node_svc: PostgresNodeService): finally: node_svc.port_manager.release_port(target_port) + @pytest.mark.parametrize( + ("source_svc", "target_svc"), + [ + pytest.param( + PostgresNodeServices.sm_remote, + PostgresNodeServices.sm_local, + id="remote_to_local", + ), + ], + ) + def test_replicate__different_os_ops( + self, + source_svc: PostgresNodeService, + target_svc: PostgresNodeService, + ): + assert source_svc.os_ops is not target_svc.os_ops + + with __class__.helper__get_node(source_svc) as master: + master.init(allow_streaming=True).start() + + with master.replicate( + target_os_ops=target_svc.os_ops, + ).start() as replica: + assert replica.os_ops is target_svc.os_ops + assert replica.host == target_svc.os_ops.host + + master.execute( + 'create table test_cross_context (val int)', + commit=True, + ) + master.execute( + 'insert into test_cross_context values (42)', + commit=True, + ) + + replica.catchup() + + res = replica.execute('select val from test_cross_context') + assert res == [(42, )] + + assert replica.source_walsender is not None + def test_synchronous_replication(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) From c673cb0e1083cb198eb8ccc4d5f0a961692ce325 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 13 Oct 2026 16:03:04 +0300 Subject: [PATCH 03/11] Test are updated (new asserts) --- tests/test_testgres_common.py | 40 +++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index abe83bb9..4af2f56a 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -239,6 +239,9 @@ def test_node_constructor__host(self): assert node2._port != node._port assert node2._os_ops is node._os_ops assert node2._port_manager is node._port_manager + + assert node2.name == C_NODE2_NAME + assert node2.base_dir == nodedir2 return def test_node_clone__target_context(self, node_svc: PostgresNodeService): @@ -253,18 +256,26 @@ def test_node_clone__target_context(self, node_svc: PostgresNodeService): name="target", base_dir=nodedir, target_os_ops=node_svc.os_ops, - target_host=node.host, + target_host="abc", target_port=target_port, ) as node2: + assert node2 is not node assert node2.os_ops is node_svc.os_ops - assert node2.host == node.host + assert node2.name == "target" + assert node2.host == "abc" assert node2.port == target_port assert node2.port_manager is None finally: node_svc.port_manager.release_port(target_port) + return + + def test_node_clone__different_os_ops( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) - def test_node_clone__different_os_ops(self): - source_svc = PostgresNodeServices.sm_local + source_svc = node_svc target_os_ops = LocalOperations() assert target_os_ops is not source_svc.os_ops @@ -283,11 +294,15 @@ def test_node_clone__different_os_ops(self): base_dir=nodedir, target_os_ops=target_os_ops, ) as node2: + assert node2 is not node assert node2.os_ops is target_os_ops + assert node2.name == "target" + assert node2.base_dir == nodedir assert node2.host == target_os_ops.host assert node2.port != node.port assert node2.port_manager is not node.port_manager assert type(node2.port_manager) is PortManager__Generic2 + return def test_node_repr(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -1940,10 +1955,18 @@ def test_backup__different_os_ops( assert backup.os_ops is target_svc.os_ops with backup.spawn_primary(destroy=False) as target: + assert target is not source + assert type(target) is PostgresNode assert target.os_ops is target_svc.os_ops assert target.host == target_svc.os_ops.host assert target.base_dir != backup.base_dir - assert target_svc.os_ops.isdir(target.data_dir) + assert target.os_ops.isdir(target.data_dir) + assert target.port_manager is not None + assert target.port_manager is not source.port_manager + + target.slow_start() + target.stop() + return def test_backup_exhaust(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -2137,6 +2160,7 @@ def test_replicate__target_context(self, node_svc: PostgresNodeService): master.init(allow_streaming=True).start() target_port = node_svc.port_manager.reserve_port() + assert target_port != master.port try: # For local operations use an equivalent host spelling to # exercise source_walsender with different host strings. A @@ -2150,12 +2174,15 @@ def test_replicate__target_context(self, node_svc: PostgresNodeService): target_os_ops=node_svc.os_ops, target_host=target_host, target_port=target_port, - ).start() as replica: + ) as replica: + assert type(replica) is PostgresNode assert replica.os_ops is node_svc.os_ops assert replica.host == target_host assert replica.port == target_port assert replica.port_manager is None + replica.start() + master.execute('create table test_target_context (val int)', commit=True) master.execute('insert into test_target_context values (42)', commit=True) @@ -2167,6 +2194,7 @@ def test_replicate__target_context(self, node_svc: PostgresNodeService): assert replica.source_walsender is not None finally: node_svc.port_manager.release_port(target_port) + return @pytest.mark.parametrize( ("source_svc", "target_svc"), From d5fb01da836706ee3c23897edd70e8d083b05cbc Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Fri, 16 Oct 2026 18:29:47 +0700 Subject: [PATCH 04/11] Add PgLab Linux fault-injection support PgLab keeps PostgreSQL lifecycle, backup and replication in Testgres while placing nodes into separate Linux network namespaces and cgroups. This provides a concrete consumer for the target execution-context replication API: replicas can use their own OsOperations, address and port while pg_basebackup runs in the target context. --- README.md | 28 ++ src/pglab.py | 1111 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1139 insertions(+) create mode 100644 src/pglab.py diff --git a/README.md b/README.md index d615d367..99457a08 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,34 @@ with testgres.get_new_node('master') as master: print(replica.execute('postgres', 'select 1')) ``` +### Linux fault injection with PgLab + +`testgres.pglab` builds Linux-isolated PostgreSQL nodes on top of the normal +Testgres lifecycle. Each PgLab node gets its own network namespace and cgroup, +while Testgres still owns `initdb`, `pg_ctl`, `pg_basebackup`, replication and +SQL access. + +This is also a concrete use case for the target execution-context arguments of +`PostgresNode.replicate()`: a replica must be created with its own +`OsOperations`, host and port, and `pg_basebackup` must run in that target +context. + +```python +from testgres.pglab import PgLab + +with PgLab(pg_bin='/path/to/postgres/bin', os_user='postgres') as lab: + primary = lab.create_primary('p1', '10.203.0.10').init().start() + replica = lab.create_replica('p1', 'r1', '10.203.0.11').start() + replica.catchup() + + lab.partition('p1', 'r1') + lab.heal_partition('p1', 'r1') +``` + +PgLab is Linux-only and currently requires root privileges because it creates +network namespaces, veth devices and cgroups. Its privileged integration suite +is run separately in CI. + ### Benchmarks Use `pgbench` through `testgres` to run quick benchmarks: diff --git a/src/pglab.py b/src/pglab.py new file mode 100644 index 00000000..8ea3c798 --- /dev/null +++ b/src/pglab.py @@ -0,0 +1,1111 @@ +"""Linux fault-injection lab built on top of Testgres. + +PgLab keeps PostgreSQL lifecycle, backup and replication in Testgres while +placing individual nodes into separate Linux network namespaces and cgroups. +It is Linux-only and currently requires root privileges. +""" + +from __future__ import annotations + +import hashlib +import ipaddress +import os +import pwd +import signal +import shutil +import stat +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from .api import get_new_node +from testgres.operations.local_ops import LocalOperations + + +class PgLabError(RuntimeError): + pass + + +def _run(cmd, check=True, capture=False): + kwargs = {"check": check, "text": True} + if capture: + kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return subprocess.run(list(cmd), **kwargs) + + +def _which(name): + path = shutil.which(name) + if not path: + raise PgLabError("required command not found: {}".format(name)) + return path + + +def _ifname(prefix, name): + # Linux IFNAMSIZ is 16 including the terminating NUL. + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:6] + return (prefix + digest)[:15] + + +@dataclass +class IOLimits: + read_bps: Optional[int] = None + write_bps: Optional[int] = None + read_iops: Optional[int] = None + write_iops: Optional[int] = None + + def as_io_max(self, major, minor): + parts = [] + if self.read_bps is not None: + parts.append("rbps={}".format(self.read_bps)) + if self.write_bps is not None: + parts.append("wbps={}".format(self.write_bps)) + if self.read_iops is not None: + parts.append("riops={}".format(self.read_iops)) + if self.write_iops is not None: + parts.append("wiops={}".format(self.write_iops)) + if not parts: + return None + return "{}:{} {}".format(major, minor, " ".join(parts)) + + +class CgroupIOBackend: + version = None + + def setup_root(self): + raise NotImplementedError + + def create_node(self, name): + raise NotImplementedError + + def apply_limits(self, cgroup, device, limits): + raise NotImplementedError + + def membership_file(self, cgroup): + raise NotImplementedError + + def kill_node(self, cgroup): + pass + + def remove_node(self, cgroup): + cgroup.rmdir() + + def cleanup_root(self): + try: + self.root.rmdir() + except OSError: + pass + + +class CgroupV2IO(CgroupIOBackend): + version = 2 + + def __init__(self, root=Path("/sys/fs/cgroup/testgres-pglab")): + self.mount = Path("/sys/fs/cgroup") + self.root = Path(root) + + def setup_root(self): + controllers = (self.mount / "cgroup.controllers").read_text().split() + if "io" not in controllers: + raise PgLabError("cgroup v2 io controller is unavailable") + subtree = self.mount / "cgroup.subtree_control" + enabled = subtree.read_text().split() + if "io" not in enabled: + try: + subtree.write_text("+io\n") + except OSError as exc: + raise PgLabError("cannot enable cgroup v2 io controller: {}".format(exc)) + self.root.mkdir(exist_ok=True) + child_subtree = self.root / "cgroup.subtree_control" + available = (self.root / "cgroup.controllers").read_text().split() + if "io" in available and "io" not in child_subtree.read_text().split(): + child_subtree.write_text("+io\n") + + def create_node(self, name): + path = self.root / name + path.mkdir(exist_ok=True) + return path + + def apply_limits(self, cgroup, device, limits): + st = os.stat(device) + line = limits.as_io_max(os.major(st.st_rdev), os.minor(st.st_rdev)) + if line: + (cgroup / "io.max").write_text(line + "\n") + + def membership_file(self, cgroup): + return cgroup / "cgroup.procs" + + def kill_node(self, cgroup): + try: + (cgroup / "cgroup.kill").write_text("1\n") + except OSError: + pass + + +class CgroupV1Blkio(CgroupIOBackend): + version = 1 + + def __init__(self, mount=Path("/sys/fs/cgroup/blkio")): + self.mount = Path(mount) + self.root = self.mount / "testgres-pglab" + + def setup_root(self): + if not self.mount.is_dir(): + raise PgLabError("cgroup v1 blkio controller is unavailable") + self.root.mkdir(exist_ok=True) + + def create_node(self, name): + path = self.root / name + path.mkdir(exist_ok=True) + return path + + def _write_limit(self, cgroup, filename, major, minor, value): + if value is not None: + (cgroup / filename).write_text("{}:{} {}\n".format(major, minor, value)) + + def apply_limits(self, cgroup, device, limits): + st = os.stat(device) + major, minor = os.major(st.st_rdev), os.minor(st.st_rdev) + self._write_limit(cgroup, "blkio.throttle.read_bps_device", major, minor, limits.read_bps) + self._write_limit(cgroup, "blkio.throttle.write_bps_device", major, minor, limits.write_bps) + self._write_limit(cgroup, "blkio.throttle.read_iops_device", major, minor, limits.read_iops) + self._write_limit(cgroup, "blkio.throttle.write_iops_device", major, minor, limits.write_iops) + + def membership_file(self, cgroup): + return cgroup / "tasks" + + +def detect_cgroup_io_backend(): + # Pure/unified v2 has cgroup.controllers at the top-level mount. + if Path("/sys/fs/cgroup/cgroup.controllers").is_file(): + return CgroupV2IO() + # Ubuntu 20.04 commonly uses hybrid mode: cgroup2 is mounted under + # /sys/fs/cgroup/unified, while blkio remains on the v1 hierarchy. + if Path("/sys/fs/cgroup/blkio").is_dir(): + return CgroupV1Blkio() + raise PgLabError("no supported cgroup I/O controller found (need v2 io or v1 blkio)") + + +class NamespaceOperations(LocalOperations): + """Local Testgres operations whose *commands* execute in a netns. + + Filesystem operations intentionally remain ordinary local operations: a Linux + network namespace shares the host filesystem. PostgreSQL utilities launched + by Testgres (initdb, pg_ctl, psql, pg_basebackup, pg_isready, ...) are wrapped + in:: + + ip netns exec ... runuser -u -- + + The wrapper enters the node cgroup before dropping privileges, so postgres + and all of its descendants inherit the cgroup. + """ + + def __init__(self, namespace, os_user, db_user, cgroup=None, cgroup_membership_file=None, pg_bin=None): + super().__init__() + self.namespace = namespace + self.os_user = os_user + self.db_user = db_user + self.pg_bin = str(pg_bin) if pg_bin is not None else None + self.cgroup = Path(cgroup) if cgroup is not None else None + self.cgroup_membership_file = ( + Path(cgroup_membership_file) if cgroup_membership_file is not None else None + ) + + # Testgres uses os_ops.username as the default database username in a + # number of places. Keep it distinct from the Unix account we use in + # runuser. + self._username = db_user + + def environ(self, name): + # target_os_ops may be asked to resolve PostgreSQL binaries while a + # replica is being cloned. Expose the PgLab-specific bin directory to + # Testgres without requiring the caller to mutate the process-wide + # PG_BIN environment variable. + if name == "PG_BIN" and self.pg_bin is not None: + return self.pg_bin + return super().environ(name) + + def mkdtemp(self, prefix=None): + assert prefix is None or type(prefix) is str + + base = "/tmp/testgres-pglab-tmp" + os.makedirs(base, mode=0o755, exist_ok=True) + + path = tempfile.mkdtemp( + prefix=prefix, + dir=base, + ) + + pw = pwd.getpwnam(self.os_user) + os.chown(path, pw.pw_uid, pw.pw_gid) + os.chmod(path, 0o700) + + return path + + def mkstemp(self, prefix=None): + assert prefix is None or type(prefix) is str + + base = "/tmp/testgres-pglab-tmp" + os.makedirs(base, mode=0o755, exist_ok=True) + + fd, path = tempfile.mkstemp( + prefix=prefix, + dir=base, + ) + os.close(fd) + + pw = pwd.getpwnam(self.os_user) + os.chown(path, pw.pw_uid, pw.pw_gid) + os.chmod(path, 0o600) + + return path + + def create_clone(self): + return self.__class__( + namespace=self.namespace, + os_user=self.os_user, + db_user=self.db_user, + cgroup=str(self.cgroup) if self.cgroup is not None else None, + cgroup_membership_file=( + str(self.cgroup_membership_file) + if self.cgroup_membership_file is not None else None + ), + pg_bin=self.pg_bin, + ) + + def _command_needs_server_cgroup(self, payload): + """Return True only for commands that may start the postgres server. + + Client utilities such as psql, pg_basebackup and pg_isready should not + be charged to the server's I/O cgroup. pg_ctl inherits the cgroup and + the postmaster it starts keeps that membership. + """ + if not payload: + return False + exe = os.path.basename(str(payload[0])) + if exe == "postgres": + return True + if exe == "pg_ctl": + # Testgres normally invokes pg_ctl with an explicit action. Only + # start/restart can create a postmaster that needs to inherit the + # node cgroup. + args = [str(x) for x in payload[1:]] + return "start" in args or "restart" in args + return False + + def _with_pg_path(self, argv): + if self.pg_bin is None: + return list(argv) + path = self.pg_bin + os.pathsep + os.environ.get("PATH", "") + return ["env", "PATH=" + path] + list(argv) + + def _wrap_command(self, cmd, shell): + if shell: + if isinstance(cmd, str): + command = cmd + else: + command = self.join_command_arguments(list(cmd)) + payload = ["sh", "-c", command] + else: + if isinstance(cmd, str): + payload = ["sh", "-c", cmd] + else: + payload = list(cmd) + + prefix = ["ip", "netns", "exec", self.namespace] + run_as_user = ["runuser", "-u", self.os_user, "--"] + self._with_pg_path(payload) + + # Most Testgres utilities are clients. They belong to the controller, + # not to the PostgreSQL server cgroup. + if self.cgroup is None or not self._command_needs_server_cgroup(payload): + return prefix + run_as_user, False + + membership = self.cgroup_membership_file + if membership is None: + raise PgLabError("cgroup membership file was not configured") + if not membership.exists(): + raise PgLabError( + "node cgroup disappeared before postgres start: {}".format(membership) + ) + + # Join the cgroup in the host namespace first, then enter the network + # namespace. Some systems do not expose the host cgroup filesystem from + # an ip-netns helper mount namespace, while cgroup membership itself is + # inherited perfectly well across ip netns exec/runuser/pg_ctl. + script = ( + 'echo $$ > "$1"; ' + 'ns="$2"; user="$3"; shift 3; ' + 'exec ip netns exec "$ns" runuser -u "$user" -- "$@"' + ) + wrapped = [ + "sh", "-c", script, "pglab-cgroup", str(membership), + self.namespace, self.os_user, + ] + self._with_pg_path(payload) + return wrapped, False + + def exec_command(self, cmd, *args, **kwargs): + shell = kwargs.pop("shell", False) + wrapped, wrapped_shell = self._wrap_command(cmd, shell) + return super().exec_command(wrapped, *args, shell=wrapped_shell, **kwargs) + + def popen(self, cmd, *args, **kwargs): + shell = kwargs.pop("shell", False) + wrapped, wrapped_shell = self._wrap_command(cmd, shell) + return super().popen(wrapped, *args, shell=wrapped_shell, **kwargs) + + +class LabNode: + """A Linux-isolated node backed by a real testgres.PostgresNode.""" + + def __init__(self, lab, name, ip, namespace, veth_host, cgroup, + pg_node, os_ops, role): + self.lab = lab + self.name = name + self.ip = ip + self.namespace = namespace + self.veth_host = veth_host + self.veth_ns = "eth0" + self.cgroup = cgroup + self.pg = pg_node + self.os_ops = os_ops + self.role = role + + def __getattr__(self, name): + # Keep the normal Testgres API pleasant to use: + # node.execute(), node.safe_psql(), node.append_conf(), ... + return getattr(self.pg, name) + + def init(self, initdb_params=None, **kwargs): + params = list(initdb_params or []) + if "-U" not in params and "--username" not in params: + params += ["-U", self.lab.db_superuser] + self.lab._ensure_node_ownership(self) + self.pg.init(initdb_params=params, **kwargs) + + # Network namespaces isolate TCP/IP but share the host filesystem. + # Without this, nodes using the same PostgreSQL port collide on + # /tmp/.s.PGSQL. even though their TCP sockets are isolated. + self.pg.append_conf(unix_socket_directories="") + + self.lab._ensure_node_ownership(self) + return self + + def start(self, *args, **kwargs): + # Testgres may lazily create logs_dir during start(). Force creation + # while the controller is root, then hand the whole node tree to the + # configured OS user before pg_ctl is run through runuser. + _ = self.pg.logs_dir + self.lab._ensure_node_ownership(self) + self.pg.start(*args, **kwargs) + return self + + def stop(self, *args, **kwargs): + self.pg.stop(*args, **kwargs) + return self + + def network_down(self): + self.lab.network_down(self.name) + return self + + def network_up(self): + self.lab.network_up(self.name) + return self + + def netem(self, **kwargs): + self.lab.netem(self.name, **kwargs) + return self + + def clear_netem(self): + self.lab.clear_netem(self.name) + return self + + def freeze(self): + self.lab.freeze(self.name) + return self + + def unfreeze(self): + self.lab.unfreeze(self.name) + return self + + +class PgLab: + """Testgres + Linux network namespaces + cgroup I/O fault injection. + + PostgreSQL lifecycle is delegated to Testgres. PgLab owns only Linux + isolation (netns/veth/bridge/cgroup) and fault injection. + """ + + def __init__( + self, + pg_bin=None, + base_dir="/tmp/testgres-pglab", + subnet="10.203.0.0/24", + bridge_name="pglab0", + bridge_ip=None, + os_user="postgres", + db_superuser=None, + keep_data=False, + ): + self.pg_bin = str(pg_bin) if pg_bin is not None else None + self.base_dir = Path(base_dir) + self.subnet = ipaddress.ip_network(subnet, strict=True) + self.bridge_name = bridge_name + self.os_user = os_user + self.db_superuser = db_superuser or os_user + self.keep_data = keep_data + self.nodes = {} # type: Dict[str, LabNode] + self._created_bridge = False + self._closed = False + self._firewall_backend = None + self._firewall_source_added = False + self._iptables_rule_added = False + + + hosts = self.subnet.hosts() + default_bridge_ip = str(next(hosts)) + self.bridge_ip = bridge_ip or default_bridge_ip + if ipaddress.ip_address(self.bridge_ip) not in self.subnet: + raise PgLabError("bridge IP is outside lab subnet: {}".format(self.bridge_ip)) + + self._preflight() + self.base_dir.mkdir(parents=True, exist_ok=True) + self._setup_cgroups() + self._ensure_bridge() + self._setup_firewall() + + def _preflight(self): + if os.name != "posix" or not Path("/proc/self/ns/net").exists(): + raise PgLabError("pglab requires Linux network namespaces") + if os.geteuid() != 0: + raise PgLabError("pglab must currently be run as root") + _which("ip") + _which("runuser") + _which("findmnt") + try: + pwd.getpwnam(self.os_user) + except KeyError as exc: + raise PgLabError("OS user does not exist: {}".format(self.os_user)) from exc + # cgroup mode is detected separately. Ubuntu 20.04 commonly uses + # hybrid mode with blkio on cgroup v1. + if self.pg_bin is not None: + required = ("initdb", "pg_ctl", "postgres", "psql", "pg_basebackup") + for exe in required: + p = Path(self.pg_bin) / exe + if not p.is_file() or not os.access(str(p), os.X_OK): + raise PgLabError("PostgreSQL executable missing/not executable: {}".format(p)) + + def _setup_cgroups(self): + self.cgroup_backend = detect_cgroup_io_backend() + self.cgroup_backend.setup_root() + self.cgroup_root = self.cgroup_backend.root + + def _ensure_bridge(self): + exists = subprocess.run( + ["ip", "link", "show", "dev", self.bridge_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode == 0 + if not exists: + _run(["ip", "link", "add", self.bridge_name, "type", "bridge"]) + self._created_bridge = True + + # The host gets an address in the lab subnet. This is important because + # testgres' Python DB driver connects from the controller namespace to + # node.host; unlike psql, execute()/connect() do not go through os_ops. + addr = "{}/{}".format(self.bridge_ip, self.subnet.prefixlen) + _run(["ip", "addr", "replace", addr, "dev", self.bridge_name]) + _run(["ip", "link", "set", self.bridge_name, "up"]) + + def _setup_firewall(self): + """Allow traffic between PgLab nodes without changing persistent rules. + + firewalld and iptables-nft can be independent forwarding layers. When + firewalld is active we add the lab subnet to the runtime trusted zone, + and independently add a temporary iptables FORWARD allow rule. + """ + source = str(self.subnet) + firewall_cmd = shutil.which("firewall-cmd") + + if firewall_cmd is not None: + state = subprocess.run( + [firewall_cmd, "--state"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if state.returncode == 0: + zone_proc = subprocess.run( + [firewall_cmd, "--get-zone-of-source={}".format(source)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + zone = zone_proc.stdout.strip() + if zone and zone != "trusted": + raise PgLabError( + "lab subnet {} already belongs to firewalld zone {!r}; " + "refusing to move an existing firewall configuration".format( + source, zone + ) + ) + if zone != "trusted": + proc = subprocess.run( + [ + firewall_cmd, + "--zone=trusted", + "--add-source={}".format(source), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode != 0: + raise PgLabError( + "failed to add lab subnet {} to firewalld trusted zone: {}".format( + source, proc.stderr.strip() + ) + ) + self._firewall_source_added = True + self._firewall_backend = "firewalld" + + # firewalld and iptables-nft are not alternatives on many developer + # hosts. Docker/libvirt may leave an iptables FORWARD base chain with + # policy DROP at nft priority 0, while firewalld evaluates forwarding + # again in a later base chain (commonly priority +10). Both layers + # therefore need a temporary allow for the lab subnet. + iptables = shutil.which("iptables") + if iptables is None: + return + + rule = [ + "FORWARD", "-s", source, "-d", source, "-j", "ACCEPT", + ] + exists = subprocess.run( + [iptables, "-C"] + rule, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ).returncode == 0 + if not exists: + _run([iptables, "-I"] + rule) + self._iptables_rule_added = True + + if self._firewall_backend is None: + self._firewall_backend = "iptables" + + def _cleanup_firewall(self): + source = str(self.subnet) + + # The two firewall layers are independent and can both be active. + # Remove only rules that this PgLab instance actually added. + if self._firewall_source_added: + firewall_cmd = shutil.which("firewall-cmd") + if firewall_cmd is not None: + subprocess.run( + [ + firewall_cmd, + "--zone=trusted", + "--remove-source={}".format(source), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self._firewall_source_added = False + + if self._iptables_rule_added: + iptables = shutil.which("iptables") + if iptables is not None: + subprocess.run( + [ + iptables, "-D", "FORWARD", + "-s", source, "-d", source, + "-j", "ACCEPT", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self._iptables_rule_added = False + + @staticmethod + def _sysfs_block_path(device): + st = os.stat(device) + if not stat.S_ISBLK(st.st_mode): + raise PgLabError("not a block device: {}".format(device)) + return Path("/sys/dev/block/{}:{}".format(os.major(st.st_rdev), os.minor(st.st_rdev))) + + @staticmethod + def _device_from_sysfs(sysdev): + # sysdev may be a /sys/dev/block/M:m symlink or a path reached through + # a device-mapper/md 'slaves' directory. The kernel exposes DEVNAME + # in uevent for both disks and partitions. + uevent = Path(sysdev) / "uevent" + if not uevent.is_file(): + raise PgLabError("cannot resolve block device from sysfs: {}".format(sysdev)) + values = {} + for line in uevent.read_text().splitlines(): + if "=" in line: + k, v = line.split("=", 1) + values[k] = v + devname = values.get("DEVNAME") + if not devname: + raise PgLabError("DEVNAME is missing in {}".format(uevent)) + return "/dev/{}".format(devname) + + def _leaf_block_devices(self, device): + """Return physical queue devices suitable for blkio throttling. + + findmnt may return a partition or a stacked device such as LVM/dm-crypt. + cgroup-v1 blkio throttling can reject such a major:minor with ENODEV. + Walk sysfs 'slaves' recursively and, for partitions, use the parent disk. + """ + start = self._sysfs_block_path(device).resolve() + found = {} + + def visit(sysdev): + sysdev = Path(sysdev).resolve() + slaves = sysdev / "slaves" + if slaves.is_dir(): + children = list(slaves.iterdir()) + if children: + for child in children: + visit(child) + return + + # A partition has its own major:minor, but blkio v1 throttling is + # most portable when applied to the whole queue-bearing disk. + if (sysdev / "partition").is_file(): + parent = sysdev.parent + if (parent / "dev").is_file(): + sysdev = parent + + dev = self._device_from_sysfs(sysdev) + st = os.stat(dev) + key = (os.major(st.st_rdev), os.minor(st.st_rdev)) + found[key] = dev + + visit(start) + return [found[k] for k in sorted(found)] + + def _backing_block_devices(self, path): + res = _run(["findmnt", "-n", "-o", "SOURCE", "-T", str(path)], capture=True) + source = res.stdout.strip() + if not source.startswith("/dev/"): + raise PgLabError( + "cannot infer block device for {}: {!r}; pass io_device".format(path, source) + ) + devices = self._leaf_block_devices(source) + if not devices: + raise PgLabError("cannot resolve backing block device for {}".format(source)) + return devices + + def _apply_io_limits(self, cgroup, devices, limits): + if isinstance(devices, (str, Path)): + devices = [str(devices)] + for device in devices: + st = os.stat(device) + if not stat.S_ISBLK(st.st_mode): + raise PgLabError("not a block device: {}".format(device)) + self.cgroup_backend.apply_limits(cgroup, device, limits) + + def _ensure_node_ownership(self, node): + pw = pwd.getpwnam(self.os_user) + base = Path(node.pg.base_dir) + base.mkdir(parents=True, exist_ok=True) + for root, dirs, files in os.walk(str(base)): + os.chown(root, pw.pw_uid, pw.pw_gid) + for d in dirs: + os.chown(os.path.join(root, d), pw.pw_uid, pw.pw_gid) + for f in files: + try: + os.chown(os.path.join(root, f), pw.pw_uid, pw.pw_gid) + except FileNotFoundError: + pass + + def create_node(self, name, ip, role="standalone", io=None, io_device=None, port=5432): + if name in self.nodes: + raise PgLabError("node already exists: {}".format(name)) + if role not in ("primary", "replica", "standalone"): + raise PgLabError("invalid role: {}".format(role)) + + addr_obj = ipaddress.ip_address(ip) + if addr_obj not in self.subnet: + raise PgLabError("{} is outside lab subnet {}".format(ip, self.subnet)) + if ip == self.bridge_ip: + raise PgLabError("{} is reserved for the pglab bridge".format(ip)) + if any(n.ip == ip for n in self.nodes.values()): + raise PgLabError("IP already in use: {}".format(ip)) + + ns = "pglab-{}".format(name) + vh = _ifname("ph", name) + peer = _ifname("pn", name) + cgroup = self.cgroup_root / name + node_base = self.base_dir / name + + created_ns = False + created_veth = False + created_cgroup = False + try: + _run(["ip", "netns", "add", ns]) + created_ns = True + _run(["ip", "link", "add", vh, "type", "veth", "peer", "name", peer]) + created_veth = True + _run(["ip", "link", "set", peer, "netns", ns]) + _run(["ip", "link", "set", vh, "master", self.bridge_name]) + _run(["ip", "link", "set", vh, "up"]) + _run(["ip", "-n", ns, "link", "set", "lo", "up"]) + _run(["ip", "-n", ns, "link", "set", peer, "name", "eth0"]) + _run(["ip", "-n", ns, "addr", "add", + "{}/{}".format(ip, self.subnet.prefixlen), "dev", "eth0"]) + _run(["ip", "-n", ns, "link", "set", "eth0", "up"]) + + node_base.mkdir(parents=True, exist_ok=True) + pw = pwd.getpwnam(self.os_user) + os.chown(str(node_base), pw.pw_uid, pw.pw_gid) + + cgroup = self.cgroup_backend.create_node(name) + created_cgroup = True + if io is not None: + devices = [io_device] if io_device else self._backing_block_devices(node_base) + self._apply_io_limits(cgroup, devices, io) + + os_ops = NamespaceOperations( + namespace=ns, + os_user=self.os_user, + db_user=self.db_superuser, + cgroup=cgroup, + cgroup_membership_file=self.cgroup_backend.membership_file(cgroup), + pg_bin=self.pg_bin, + ) + + kwargs = { + "name": name, + "base_dir": str(node_base), + "port": int(port), + "host": ip, + "os_ops": os_ops, + } + if self.pg_bin is not None: + kwargs["bin_dir"] = self.pg_bin + pg_node = get_new_node(**kwargs) + + node = LabNode( + lab=self, + name=name, + ip=ip, + namespace=ns, + veth_host=vh, + cgroup=cgroup, + pg_node=pg_node, + os_ops=os_ops, + role=role, + ) + self.nodes[name] = node + return node + except Exception: + if created_veth: + subprocess.run(["ip", "link", "del", vh], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if created_ns: + subprocess.run(["ip", "netns", "del", ns], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if created_cgroup: + try: + cgroup.rmdir() + except OSError: + pass + if not self.keep_data: + shutil.rmtree(str(node_base), ignore_errors=True) + raise + + def create_primary(self, name, ip, **kwargs): + return self.create_node(name, ip, role="primary", **kwargs) + + def create_replica(self, primary_name, name, ip, **kwargs): + """Create a physical replica in its own network namespace. + + Current Testgres can run pg_basebackup directly in ``target_os_ops``. + PgLab creates the namespace/cgroup first, then asks Testgres to create + the replica through that execution context. The base backup therefore + originates from the replica namespace instead of being copied into it + afterwards. + """ + primary = self.nodes[primary_name] + replica_shell = self.create_node(name, ip, role="replica", **kwargs) + + target_ops = replica_shell.os_ops + target_port = replica_shell.pg.port + target_base = replica_shell.pg.base_dir + + # create_node() constructs a placeholder PostgresNode so PgLab can + # allocate the namespace, cgroup, address, port and storage location + # before replication. Reuse that base directory for the target-side + # base backup; this keeps replica data under PgLab.base_dir (important + # when PGLAB_TEST_ROOT selects a block-backed filesystem for I/O tests). + try: + spawned = primary.pg.replicate( + name=name, + target_os_ops=target_ops, + target_host=ip, + target_port=target_port, + username=self.db_superuser, + base_dir=target_base, + ) + except Exception: + self.remove_node(name) + raise + + replica_shell.pg = spawned + self._ensure_node_ownership(replica_shell) + return replica_shell + + def network_down(self, name): + node = self.nodes[name] + _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "down"]) + + def network_up(self, name): + node = self.nodes[name] + _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "up"]) + + @staticmethod + def _percent(value, name): + value = float(value) + if value < 0.0 or value > 100.0: + raise PgLabError("{} must be between 0 and 100".format(name)) + return value + + def netem( + self, + name, + delay_ms=None, + jitter_ms=None, + loss_pct=None, + duplicate_pct=None, + corrupt_pct=None, + reorder_pct=None, + rate_kbit=None, + ): + """Replace the node's root qdisc with a tc-netem fault profile. + + The profile is applied to traffic leaving the node's eth0 inside its + network namespace. Calling netem() again replaces the previous profile. + + Examples: + lab.netem("r1", delay_ms=100) + lab.netem("r1", delay_ms=100, jitter_ms=20, loss_pct=5) + lab.netem("r1", rate_kbit=1024) + lab.clear_netem("r1") + """ + node = self.nodes[name] + _which("tc") + + args = [ + "ip", "netns", "exec", node.namespace, + "tc", "qdisc", "replace", "dev", node.veth_ns, + "root", "netem", + ] + + have_fault = False + + if delay_ms is not None: + delay_ms = float(delay_ms) + if delay_ms < 0: + raise PgLabError("delay_ms must be >= 0") + args += ["delay", "{}ms".format(delay_ms)] + have_fault = True + + if jitter_ms is not None: + jitter_ms = float(jitter_ms) + if jitter_ms < 0: + raise PgLabError("jitter_ms must be >= 0") + args += ["{}ms".format(jitter_ms)] + elif jitter_ms is not None: + raise PgLabError("jitter_ms requires delay_ms") + + if loss_pct is not None: + loss_pct = self._percent(loss_pct, "loss_pct") + args += ["loss", "{}%".format(loss_pct)] + have_fault = True + + if duplicate_pct is not None: + duplicate_pct = self._percent(duplicate_pct, "duplicate_pct") + args += ["duplicate", "{}%".format(duplicate_pct)] + have_fault = True + + if corrupt_pct is not None: + corrupt_pct = self._percent(corrupt_pct, "corrupt_pct") + args += ["corrupt", "{}%".format(corrupt_pct)] + have_fault = True + + if reorder_pct is not None: + reorder_pct = self._percent(reorder_pct, "reorder_pct") + if delay_ms is None: + raise PgLabError("reorder_pct requires delay_ms") + args += ["reorder", "{}%".format(reorder_pct)] + have_fault = True + + if rate_kbit is not None: + rate_kbit = int(rate_kbit) + if rate_kbit <= 0: + raise PgLabError("rate_kbit must be > 0") + args += ["rate", "{}kbit".format(rate_kbit)] + have_fault = True + + if not have_fault: + raise PgLabError("netem requires at least one fault parameter") + + _run(args) + + def clear_netem(self, name): + """Remove the root qdisc fault profile from a node, if present.""" + node = self.nodes[name] + if shutil.which("tc") is None: + return + subprocess.run( + [ + "ip", "netns", "exec", node.namespace, + "tc", "qdisc", "del", "dev", node.veth_ns, "root", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + def _node_cgroup_pids(self, node): + membership = Path(node.os_ops.cgroup_membership_file) + try: + lines = membership.read_text().splitlines() + except FileNotFoundError: + return [] + pids = [] + for line in lines: + line = line.strip() + if line.isdigit(): + pids.append(int(line)) + return sorted(set(pids)) + + def freeze(self, name): + """SIGSTOP every task currently in the node's PostgreSQL cgroup. + + Unlike a network partition this leaves sockets and processes alive, but + the PostgreSQL node stops getting CPU time. This models a wedged VM, + scheduler stall, or process group that has stopped responding. + """ + node = self.nodes[name] + pids = self._node_cgroup_pids(node) + if not pids: + raise PgLabError("node cgroup has no processes to freeze: {}".format(name)) + + stopped = [] + try: + for pid in pids: + try: + os.kill(pid, signal.SIGSTOP) + stopped.append(pid) + except ProcessLookupError: + pass + except Exception: + # Do not leave a partially frozen node after a failed injection. + for pid in stopped: + try: + os.kill(pid, signal.SIGCONT) + except ProcessLookupError: + pass + raise + + def unfreeze(self, name): + """SIGCONT all tasks currently present in the node cgroup.""" + node = self.nodes[name] + for pid in self._node_cgroup_pids(node): + try: + os.kill(pid, signal.SIGCONT) + except ProcessLookupError: + pass + + def partition(self, left, right): + lnode = self.nodes[left] + rnode = self.nodes[right] + _run(["ip", "-n", lnode.namespace, "route", "replace", "blackhole", + "{}/32".format(rnode.ip)]) + _run(["ip", "-n", rnode.namespace, "route", "replace", "blackhole", + "{}/32".format(lnode.ip)]) + + def heal_partition(self, left, right): + lnode = self.nodes[left] + rnode = self.nodes[right] + subprocess.run(["ip", "-n", lnode.namespace, "route", "del", "blackhole", + "{}/32".format(rnode.ip)], check=False) + subprocess.run(["ip", "-n", rnode.namespace, "route", "del", "blackhole", + "{}/32".format(lnode.ip)], check=False) + + def set_io_limits(self, name, limits, io_device=None): + node = self.nodes[name] + devices = [io_device] if io_device else self._backing_block_devices(Path(node.pg.base_dir)) + self._apply_io_limits(node.cgroup, devices, limits) + + def remove_node(self, name): + node = self.nodes.get(name) + if node is None: + return + + # A frozen PostgreSQL cannot cooperate with pg_ctl stop/cleanup. + # Restore process scheduling and remove traffic shaping first. + try: + self.unfreeze(name) + except Exception: + pass + try: + self.clear_netem(name) + except Exception: + pass + + # PgLab removes the whole node directory itself below. Using + # PostgresNode.cleanup() here would perform another stop attempt first. + # That is noisy for two intentional PgLab states: an uninitialized node + # and a postmaster that a fault-injection test already killed. stop() + # itself always stops Testgres' log reader in its finally block, so call + # it directly, tolerate the expected failure, and release Testgres-owned + # resources before removing the isolation objects. + try: + node.pg.stop() + except Exception: + pass + try: + node.pg.release_resources() + except Exception: + pass + + subprocess.run(["ip", "netns", "del", node.namespace], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(["ip", "link", "del", node.veth_host], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.cgroup_backend.kill_node(node.cgroup) + try: + self.cgroup_backend.remove_node(node.cgroup) + except OSError: + pass + if not self.keep_data: + shutil.rmtree(str(self.base_dir / name), ignore_errors=True) + del self.nodes[name] + + def close(self): + if self._closed: + return + for name in list(self.nodes.keys()): + self.remove_node(name) + self._cleanup_firewall() + if self._created_bridge: + subprocess.run(["ip", "link", "del", self.bridge_name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.cgroup_backend.cleanup_root() + if not self.keep_data: + try: + self.base_dir.rmdir() + except OSError: + pass + self._closed = True + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + self.close() + return False From 29d8f98093d00054b3998f090d54633d01ac950b Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Fri, 16 Oct 2026 18:30:07 +0700 Subject: [PATCH 05/11] Run PgLab integration tests in CI Add the PgLab regression suite to Testgres tests and run the privileged Linux scenarios in a dedicated Ubuntu job. The normal test matrix only runs the API compatibility checks. Destructive netns/cgroup tests are opt-in through TESTGRES_PGLAB_TESTS and are enabled by the dedicated CI job. --- .github/workflows/ci.yml | 70 +++++ tests/helpers/pglab_test_config.py | 25 ++ tests/test_pglab.py | 245 +++++++++++++++ tests/test_pglab_compat.py | 29 ++ tests/test_pglab_faults.py | 393 ++++++++++++++++++++++++ tests/test_pglab_netem.py | 306 +++++++++++++++++++ tests/test_pglab_netem_effects.py | 469 +++++++++++++++++++++++++++++ 7 files changed, 1537 insertions(+) create mode 100644 tests/helpers/pglab_test_config.py create mode 100644 tests/test_pglab.py create mode 100644 tests/test_pglab_compat.py create mode 100644 tests/test_pglab_faults.py create mode 100644 tests/test_pglab_netem.py create mode 100644 tests/test_pglab_netem_effects.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2121fe92..84357173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,76 @@ jobs: name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} path: "${{ env.RUN_CFG__LOGS_DIR }}/" + test-pglab: + runs-on: ubuntu-24.04 + needs: build-check + name: "test: pglab | ubuntu-24.04 | pg16" + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python 3.12 + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: 'pip' + + - name: Install PostgreSQL and PgLab system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + postgresql-16 \ + postgresql-client-16 \ + libpq-dev \ + iproute2 \ + iputils-ping \ + iptables \ + util-linux + + - name: Install Python test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: PgLab preflight + run: | + set -eux + "$(pg_config --bindir)/postgres" --version + pg_config --bindir + findmnt -n -o SOURCE,FSTYPE -T /var/tmp + mount | grep cgroup || true + cat /sys/fs/cgroup/cgroup.controllers + cat /sys/fs/cgroup/cgroup.subtree_control || true + + - name: Run privileged PgLab tests + run: | + set -eux + mkdir -p "${{ github.workspace }}/logs-pglab" + PYTHON="$(command -v python)" + PG_BIN="$(pg_config --bindir)" + sudo env \ + PATH="$PATH" \ + TESTGRES_PGLAB_TESTS=1 \ + PG_BIN="$PG_BIN" \ + PGLAB_OS_USER="$USER" \ + PGLAB_DB_USER=postgres \ + PGLAB_TEST_ROOT=/var/tmp \ + TEST_CFG__LOG_DIR="${{ github.workspace }}/logs-pglab" \ + "$PYTHON" -m pytest -vv -s \ + tests/test_pglab_compat.py \ + tests/test_pglab.py \ + tests/test_pglab_faults.py \ + tests/test_pglab_netem.py \ + tests/test_pglab_netem_effects.py + + - name: Upload PgLab logs + uses: actions/upload-artifact@v7 + if: always() + with: + name: testgres--pglab-logs--id${{ github.run_id }} + path: logs-pglab/ + test-macos-14: runs-on: macos-14 needs: build-check diff --git a/tests/helpers/pglab_test_config.py b/tests/helpers/pglab_test_config.py new file mode 100644 index 00000000..0e7add01 --- /dev/null +++ b/tests/helpers/pglab_test_config.py @@ -0,0 +1,25 @@ +import os +from pathlib import Path + +import pytest + + +PG_BIN = os.environ.get("PG_BIN") +OS_USER = os.environ.get("PGLAB_OS_USER", os.environ.get("SUDO_USER", "postgres")) +DB_USER = os.environ.get("PGLAB_DB_USER", "postgres") +TEST_ROOT = Path(os.environ.get("PGLAB_TEST_ROOT", "/var/tmp")) + +PGLAB_INTEGRATION_MARKS = [ + pytest.mark.skipif( + os.environ.get("TESTGRES_PGLAB_TESTS") != "1", + reason="set TESTGRES_PGLAB_TESTS=1 to run privileged PgLab tests", + ), + pytest.mark.skipif( + os.geteuid() != 0, + reason="PgLab integration tests require root", + ), + pytest.mark.skipif( + not PG_BIN, + reason="PG_BIN must point to the PostgreSQL bin directory", + ), +] diff --git a/tests/test_pglab.py b/tests/test_pglab.py new file mode 100644 index 00000000..514301ff --- /dev/null +++ b/tests/test_pglab.py @@ -0,0 +1,245 @@ +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest + +from src.pglab import PgLab, IOLimits +from .helpers.pglab_test_config import ( + DB_USER, + OS_USER, + PG_BIN, + PGLAB_INTEGRATION_MARKS, + TEST_ROOT, +) + + +pytestmark = PGLAB_INTEGRATION_MARKS + + +def run(*args, check=True): + return subprocess.run( + [str(a) for a in args], + check=check, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def netns_exists(name): + r = run("ip", "netns", "list") + return any(line.split()[0] == name for line in r.stdout.splitlines() if line.strip()) + + +def firewall_source_zone(subnet): + fw = shutil.which("firewall-cmd") + if fw is None: + return None + if run(fw, "--state", check=False).returncode != 0: + return None + r = run(fw, "--get-zone-of-source={}".format(subnet), check=False) + if r.returncode != 0: + return "" + return r.stdout.strip() + + +@pytest.fixture +def lab(): + # Keep names short: Linux IFNAMSIZ is 16 bytes including NUL. + token = uuid.uuid4().hex[:5] + bridge = "plt" + token + base = str(TEST_ROOT / ("pglab-pytest-" + token)) + + instance = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet="10.231.{}.0/24".format(int(token[:2], 16) % 200 + 20), + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + try: + yield instance + finally: + instance.close() + shutil.rmtree(base, ignore_errors=True) + + +def test_primary_starts_in_netns_and_uses_tcp(lab): + primary = lab.create_primary("p1", "10.231.{}.10".format(lab.subnet.network_address.packed[2])) + primary.init().start() + + assert primary.execute("select 1") == [(1,)] + expected = primary.ip + assert primary.execute( + "select host(inet_server_addr()) = '{}'".format(expected) + ) == [(True,)] + + # We deliberately disable the shared host Unix socket, because netns does + # not isolate the filesystem and every node may use TCP port 5432. + assert primary.execute("show unix_socket_directories") == [("",)] + + +def test_replication_and_catchup(lab): + third = lab.subnet.network_address.packed[2] + primary = lab.create_primary("p1", "10.231.{}.10".format(third)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", "10.231.{}.11".format(third)) + replica.start() + replica.catchup() + + assert replica.execute("select pg_is_in_recovery()") == [(True,)] + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1), (2), (3)") + replica.catchup() + + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] + + rows = primary.execute(""" + select application_name, state, host(client_addr) + from pg_stat_replication + where application_name = 'r1' + """) + assert rows == [("r1", "streaming", replica.ip)] + + +def test_replica_target_port_and_storage_root(lab): + third = lab.subnet.network_address.packed[2] + primary = lab.create_primary("p1", "10.231.{}.10".format(third), port=5432) + primary.init().start() + + replica = lab.create_replica( + "p1", + "r1", + "10.231.{}.11".format(third), + port=5433, + ) + + assert replica.port == 5433 + assert Path(replica.pg.base_dir) == lab.base_dir / "r1" + + replica.start() + replica.catchup() + + assert replica.execute("show port") == [("5433",)] + assert replica.execute("select pg_is_in_recovery()") == [(True,)] + + +def test_partition_and_heal(lab): + third = lab.subnet.network_address.packed[2] + primary = lab.create_primary("p1", "10.231.{}.10".format(third)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", "10.231.{}.11".format(third)) + replica.start() + replica.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + assert replica.execute("select count(*) from t") == [(1,)] + + lab.partition("p1", "r1") + primary.execute("insert into t values (2)") + + # Controller -> replica connectivity remains available; only node-to-node + # traffic is blackholed. + assert replica.execute("select count(*) from t") == [(1,)] + + # The sender should stop being streaming after the partition has had a + # chance to break the connection. Don't make timing the essence of this + # test; the important assertion is that replay cannot advance. + lab.heal_partition("p1", "r1") + replica.catchup() + + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + +def test_postgres_backend_is_in_node_cgroup(lab): + third = lab.subnet.network_address.packed[2] + primary = lab.create_primary( + "p1", + "10.231.{}.10".format(third), + io=IOLimits(read_bps=50 * 1024 * 1024), + ) + primary.init().start() + + backend_pid = primary.execute("select pg_backend_pid()")[0][0] + membership = Path(primary.os_ops.cgroup_membership_file) + + assert membership.exists() + + pids = { + int(line) + for line in membership.read_text().splitlines() + if line.strip().isdigit() + } + assert backend_pid in pids + + +def test_io_limit_is_programmed(lab): + third = lab.subnet.network_address.packed[2] + limit = 32 * 1024 * 1024 + + primary = lab.create_primary( + "p1", + "10.231.{}.10".format(third), + io=IOLimits(read_bps=limit), + ) + primary.init().start() + + cg = Path(primary.cgroup) + version = lab.cgroup_backend.version + + if version == 2: + text = (cg / "io.max").read_text() + assert "rbps={}".format(limit) in text + elif version == 1: + text = (cg / "blkio.throttle.read_bps_device").read_text() + assert any( + line.split()[-1] == str(limit) + for line in text.splitlines() + if line.strip() + ) + else: + pytest.fail("unsupported cgroup backend version: {!r}".format(version)) + + +def test_cleanup_removes_namespace_bridge_and_runtime_firewall(): + token = uuid.uuid4().hex[:5] + bridge = "plc" + token + base = str(TEST_ROOT / ("pglab-cleanup-" + token)) + subnet = "10.232.{}.0/24".format(int(token[:2], 16) % 200 + 20) + + zone_before = firewall_source_zone(subnet) + + lab = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + + node = lab.create_primary("cleanup", str(list(lab.subnet.hosts())[9])) + node.init().start() + namespace = node.namespace + + assert netns_exists(namespace) + assert run("ip", "link", "show", "dev", bridge, check=False).returncode == 0 + + lab.close() + + assert not netns_exists(namespace) + assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 + assert not Path(base).exists() + + # PgLab must not remove a pre-existing user firewalld assignment. If there + # was no assignment before, there must not be one left afterwards. + if zone_before == "": + assert firewall_source_zone(subnet) == "" diff --git a/tests/test_pglab_compat.py b/tests/test_pglab_compat.py new file mode 100644 index 00000000..23862692 --- /dev/null +++ b/tests/test_pglab_compat.py @@ -0,0 +1,29 @@ +import inspect + +from src.pglab import NamespaceOperations +from src.backup import NodeBackup +from src.node import PostgresNode + + +def test_testgres_target_execution_context_api(): + replicate = inspect.signature(PostgresNode.replicate).parameters + assert "target_os_ops" in replicate + assert "target_host" in replicate + assert "target_port" in replicate + + backup_init = inspect.signature(NodeBackup.__init__).parameters + assert "os_ops" in backup_init + + spawn_replica = inspect.signature(NodeBackup.spawn_replica).parameters + assert "target_host" in spawn_replica + assert "target_port" in spawn_replica + + +def test_namespace_operations_exposes_explicit_pg_bin(): + ops = NamespaceOperations( + namespace="pglab-test-unused", + os_user="postgres", + db_user="postgres", + pg_bin="/opt/postgresql/bin", + ) + assert ops.environ("PG_BIN") == "/opt/postgresql/bin" diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py new file mode 100644 index 00000000..ecbb2a63 --- /dev/null +++ b/tests/test_pglab_faults.py @@ -0,0 +1,393 @@ +import os +import shutil +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +from src.pglab import PgLab, IOLimits +from .helpers.pglab_test_config import ( + DB_USER, + OS_USER, + PG_BIN, + PGLAB_INTEGRATION_MARKS, + TEST_ROOT, +) + + +pytestmark = PGLAB_INTEGRATION_MARKS + + +def run(*args, check=True): + return subprocess.run( + [str(a) for a in args], + check=check, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + if fn(): + return + except Exception as exc: + last = exc + time.sleep(interval) + if last is not None: + raise AssertionError("{} not reached; last error: {!r}".format(desc, last)) + raise AssertionError("{} not reached".format(desc)) + + +def make_lab(prefix="flt", subnet_octet=None): + token = uuid.uuid4().hex[:5] + bridge = (prefix + token)[:15] + base = str(TEST_ROOT / ("pglab-faults-" + token)) + if subnet_octet is None: + subnet_octet = int(token[:2], 16) % 180 + 40 + subnet = "10.233.{}.0/24".format(subnet_octet) + + return PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + + +@pytest.fixture +def lab(): + instance = make_lab() + try: + yield instance + finally: + instance.close() + + +def node_ip(lab, hostno): + return str(list(lab.subnet.hosts())[hostno - 1]) + + +def pg_stat_replication(primary): + return primary.execute(""" + select application_name, state, host(client_addr) + from pg_stat_replication + order by application_name + """) + + +def test_two_replicas_same_port_and_independent_catchup(lab): + primary = lab.create_primary("p1", node_ip(lab, 10), port=5432) + primary.init().start() + + r1 = lab.create_replica("p1", "r1", node_ip(lab, 11), port=5432) + r2 = lab.create_replica("p1", "r2", node_ip(lab, 12), port=5432) + + r1.start() + r2.start() + r1.catchup() + r2.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1), (2), (3)") + r1.catchup() + r2.catchup() + + assert r1.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] + assert r2.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] + + rows = pg_stat_replication(primary) + assert rows == [ + ("r1", "streaming", r1.ip), + ("r2", "streaming", r2.ip), + ] + + assert primary.port == r1.port == r2.port == 5432 + + +def test_replica_restart_rejoins_streaming(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + replica.stop() + primary.execute("insert into t values (2), (3)") + + replica.start() + replica.catchup() + + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] + + wait_until( + lambda: pg_stat_replication(primary) == [ + ("r1", "streaming", replica.ip) + ], + desc="replica streaming after restart", + ) + + +def test_primary_restart_replica_recovers(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + primary.stop() + + # Replica remains queryable in recovery while upstream is unavailable. + assert replica.execute("select pg_is_in_recovery()") == [(True,)] + assert replica.execute("select array_agg(i order by i) from t") == [([1],)] + + primary.start() + primary.execute("insert into t values (2)") + replica.catchup() + + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + +def test_network_down_up_recovers_streaming(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + replica.network_down() + primary.execute("insert into t values (2)") + + # Controller may lose direct reachability to the replica while eth0 is down, + # so do not query it here. Restore networking first. + replica.network_up() + + wait_until( + lambda: run( + "ip", "netns", "exec", replica.namespace, + "ping", "-c", "1", "-W", "1", primary.ip, + check=False, + ).returncode == 0, + timeout=10, + desc="replica network restored", + ) + + replica.catchup() + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + +def test_live_io_limit_change(lab): + first = 64 * 1024 * 1024 + second = 8 * 1024 * 1024 + + primary = lab.create_primary( + "p1", + node_ip(lab, 10), + io=IOLimits(read_bps=first), + ) + primary.init().start() + + lab.set_io_limits("p1", IOLimits(read_bps=second)) + + cg = Path(primary.cgroup) + if lab.cgroup_backend.version == 2: + text = (cg / "io.max").read_text() + assert "rbps={}".format(second) in text + assert "rbps={}".format(first) not in text + elif lab.cgroup_backend.version == 1: + text = (cg / "blkio.throttle.read_bps_device").read_text() + values = [ + line.split()[-1] + for line in text.splitlines() + if line.strip() + ] + assert str(second) in values + else: + pytest.fail("unsupported cgroup backend") + + +def test_exception_inside_context_cleans_everything(): + token = uuid.uuid4().hex[:5] + bridge = ("boom" + token)[:15] + base = str(TEST_ROOT / ("pglab-boom-" + token)) + subnet = "10.234.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + namespace = None + with pytest.raises(RuntimeError, match="boom"): + with PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) as lab: + p = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) + namespace = p.namespace + p.init().start() + raise RuntimeError("boom") + + assert namespace is not None + assert namespace not in run("ip", "netns", "list").stdout + assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 + assert not Path(base).exists() + + +def test_keep_data_preserves_pgdata(): + token = uuid.uuid4().hex[:5] + bridge = ("keep" + token)[:15] + base = TEST_ROOT / ("pglab-keep-" + token) + subnet = "10.236.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + lab = PgLab( + pg_bin=PG_BIN, + base_dir=str(base), + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + keep_data=True, + ) + + primary = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) + primary.init().start() + primary.execute("create table keep_me(i int)") + + pgdata = Path(primary.pg.data_dir) + lab.close() + + try: + assert pgdata.is_dir() + assert (pgdata / "PG_VERSION").is_file() + finally: + shutil.rmtree(base, ignore_errors=True) + + +def test_cleanup_when_postgres_was_sigkilled(capsys): + lab = make_lab(prefix="kill") + bridge = lab.bridge_name + base = lab.base_dir + + p = lab.create_primary("p1", node_ip(lab, 10)) + p.init().start() + + postmaster_pid = p.execute("select pg_postmaster_start_time() is not null, pg_backend_pid()")[0][1] + + # Find actual postmaster pid from postmaster.pid, then kill -9 it. + pgdata = Path(p.pg.data_dir) + lines = (pgdata / "postmaster.pid").read_text().splitlines() + actual_postmaster = int(lines[0]) + + os.kill(actual_postmaster, 9) + + wait_until( + lambda: not Path("/proc/{}".format(actual_postmaster)).exists(), + timeout=10, + desc="postmaster SIGKILL", + ) + + # close() must tolerate the already-dead server and still clean isolation. + lab.close() + + captured = capsys.readouterr() + assert "cannot stop node" not in captured.err + assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 + assert not Path(base).exists() + + +def test_recreate_same_lab_after_close(): + token = uuid.uuid4().hex[:5] + bridge = ("same" + token)[:15] + base = str(TEST_ROOT / ("pglab-same-" + token)) + subnet = "10.235.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + for cycle in (1, 2): + with PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) as lab: + p = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) + p.init().start() + assert p.execute("select {}".format(cycle)) == [(cycle,)] + + assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 + assert not Path(base).exists() + + +def test_partition_only_one_replica(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + r1 = lab.create_replica("p1", "r1", node_ip(lab, 11)) + r2 = lab.create_replica("p1", "r2", node_ip(lab, 12)) + r1.start() + r2.start() + r1.catchup() + r2.catchup() + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + r1.catchup() + r2.catchup() + + lab.partition("p1", "r1") + primary.execute("insert into t values (2)") + + # r2 should continue normally while r1 is isolated. + r2.catchup() + assert r2.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + # r1 is still queryable from the controller and should not have replayed 2. + assert r1.execute("select array_agg(i order by i) from t") == [([1],)] + + lab.heal_partition("p1", "r1") + r1.catchup() + assert r1.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + +def test_replication_survives_repeated_partition_cycles(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + + primary.execute("create table t(i int primary key)") + + for i in range(1, 6): + lab.partition("p1", "r1") + primary.execute("insert into t values ({})".format(i)) + lab.heal_partition("p1", "r1") + replica.catchup() + + assert replica.execute( + "select coalesce(max(i), 0), count(*) from t" + ) == [(i, i)] diff --git a/tests/test_pglab_netem.py b/tests/test_pglab_netem.py new file mode 100644 index 00000000..6d905593 --- /dev/null +++ b/tests/test_pglab_netem.py @@ -0,0 +1,306 @@ +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +from src.pglab import PgLab, PgLabError +from .helpers.pglab_test_config import ( + DB_USER, + OS_USER, + PG_BIN, + PGLAB_INTEGRATION_MARKS, + TEST_ROOT, +) + + +pytestmark = PGLAB_INTEGRATION_MARKS + + +def run(*args, check=True): + return subprocess.run( + [str(a) for a in args], + check=check, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + if fn(): + return + except Exception as exc: + last = exc + time.sleep(interval) + if last is not None: + raise AssertionError("{} not reached; last error: {!r}".format(desc, last)) + raise AssertionError("{} not reached".format(desc)) + + +@pytest.fixture +def lab(): + token = uuid.uuid4().hex[:5] + bridge = ("net" + token)[:15] + base = str(TEST_ROOT / ("pglab-netem-" + token)) + subnet = "10.236.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + instance = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + try: + yield instance + finally: + instance.close() + + +def node_ip(lab, hostno): + return str(list(lab.subnet.hosts())[hostno - 1]) + + +def make_pair(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + return primary, replica + + +def qdisc(node): + return run( + "ip", "netns", "exec", node.namespace, + "tc", "qdisc", "show", "dev", node.veth_ns, + ).stdout + + +def postmaster_pid(node): + return int((Path(node.pg.data_dir) / "postmaster.pid").read_text().splitlines()[0]) + + +def proc_state(pid): + for line in Path("/proc/{}/status".format(pid)).read_text().splitlines(): + if line.startswith("State:"): + return line.split()[1] + raise AssertionError("no State in /proc/{}/status".format(pid)) + + +def test_netem_delay_is_installed_and_cleared(lab): + _, replica = make_pair(lab) + + replica.netem(delay_ms=150, jitter_ms=20) + + text = qdisc(replica) + assert "netem" in text + assert "delay" in text + assert "150" in text + + replica.clear_netem() + assert "netem" not in qdisc(replica) + + +def test_replication_survives_latency_and_loss(lab): + primary, replica = make_pair(lab) + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + # Large enough to exercise retransmits without making the test excessively + # slow. The fault applies to replica egress, including TCP ACKs. + replica.netem(delay_ms=80, jitter_ms=20, loss_pct=2) + + primary.execute( + "insert into t select i from generate_series(2, 1001) as g(i)" + ) + + # Avoid an aggressive Testgres catchup timeout while netem is active. + time.sleep(1.0) + replica.clear_netem() + replica.catchup() + + assert replica.execute("select count(*), max(i) from t") == [(1001, 1001)] + + +def test_netem_total_loss_then_recovery(lab): + primary, replica = make_pair(lab) + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + replica.netem(loss_pct=100) + primary.execute("insert into t values (2), (3)") + + assert "loss 100%" in qdisc(replica) + + replica.clear_netem() + replica.catchup() + + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] + + +def test_netem_can_be_replaced_live(lab): + _, replica = make_pair(lab) + + replica.netem(delay_ms=50) + first = qdisc(replica) + assert "50" in first + + replica.netem(delay_ms=250, loss_pct=1) + second = qdisc(replica) + assert "250" in second + assert "loss 1%" in second + + replica.clear_netem() + + +def test_netem_validation(lab, capsys): + node = lab.create_primary("p1", node_ip(lab, 10)) + + with pytest.raises(PgLabError): + node.netem() + + with pytest.raises(PgLabError): + node.netem(loss_pct=101) + + with pytest.raises(PgLabError): + node.netem(jitter_ms=10) + + with pytest.raises(PgLabError): + node.netem(reorder_pct=10) + + # Closing an intentionally uninitialized node should not leak Testgres' + # internal stop diagnostic to the test output. + lab.close() + captured = capsys.readouterr() + assert "cannot stop node" not in captured.err + + +def test_freeze_replica_then_resume_and_catchup(lab): + primary, replica = make_pair(lab) + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + pid = postmaster_pid(replica) + replica.freeze() + + wait_until( + lambda: proc_state(pid) in ("T", "t"), + timeout=5, + desc="replica postmaster stopped", + ) + + primary.execute("insert into t values (2), (3), (4)") + + replica.unfreeze() + + wait_until( + lambda: proc_state(pid) not in ("T", "t"), + timeout=5, + desc="replica postmaster resumed", + ) + + replica.catchup() + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3, 4],)] + + +def test_freeze_primary_then_resume(lab): + primary, replica = make_pair(lab) + + primary.execute("create table t(i int primary key)") + primary.execute("insert into t values (1)") + replica.catchup() + + pid = postmaster_pid(primary) + primary.freeze() + + wait_until( + lambda: proc_state(pid) in ("T", "t"), + timeout=5, + desc="primary postmaster stopped", + ) + + # Replica remains readable while upstream is frozen. + assert replica.execute("select array_agg(i order by i) from t") == [([1],)] + + primary.unfreeze() + wait_until( + lambda: proc_state(pid) not in ("T", "t"), + timeout=5, + desc="primary postmaster resumed", + ) + + primary.execute("insert into t values (2)") + replica.catchup() + assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] + + +def test_close_unfreezes_before_cleanup(): + token = uuid.uuid4().hex[:5] + bridge = ("frz" + token)[:15] + base = str(TEST_ROOT / ("pglab-frozen-close-" + token)) + subnet = "10.237.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + lab = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + + node = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) + node.init().start() + node.freeze() + + # Regression test: close() must not hang trying to pg_ctl-stop SIGSTOPed PG. + started = time.monotonic() + lab.close() + elapsed = time.monotonic() - started + + assert elapsed < 15 + assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 + assert not Path(base).exists() + + +def test_close_cleans_netem(): + token = uuid.uuid4().hex[:5] + bridge = ("tc" + token)[:15] + base = str(TEST_ROOT / ("pglab-netem-close-" + token)) + subnet = "10.238.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + lab = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + + node = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) + node.init().start() + node.netem(delay_ms=100, loss_pct=1) + assert "netem" in qdisc(node) + + lab.close() + + # Namespace deletion itself removes the qdisc; the important thing is that + # cleanup succeeds while a netem profile is active. + assert node.namespace not in run("ip", "netns", "list").stdout + assert not Path(base).exists() diff --git a/tests/test_pglab_netem_effects.py b/tests/test_pglab_netem_effects.py new file mode 100644 index 00000000..7215b630 --- /dev/null +++ b/tests/test_pglab_netem_effects.py @@ -0,0 +1,469 @@ +import os +import re +import shutil +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +from src.pglab import PgLab +from .helpers.pglab_test_config import ( + DB_USER, + OS_USER, + PG_BIN, + PGLAB_INTEGRATION_MARKS, + TEST_ROOT, +) + + +PSQL = str(Path(PG_BIN) / "psql") if PG_BIN else None + +pytestmark = PGLAB_INTEGRATION_MARKS + [ + pytest.mark.skipif(shutil.which("tc") is None, reason="tc/iproute2 is required"), +] + + +def run(*args, check=True, timeout=None, env=None, stdout=subprocess.PIPE): + full_env = os.environ.copy() + full_env["LC_ALL"] = "C" + if env: + full_env.update(env) + + return subprocess.run( + [str(a) for a in args], + check=check, + text=True, + stdout=stdout, + stderr=subprocess.PIPE, + timeout=timeout, + env=full_env, + ) + + +def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + if fn(): + return + except Exception as exc: + last = exc + time.sleep(interval) + + if last is not None: + raise AssertionError( + "{} not reached; last error: {!r}".format(desc, last) + ) + raise AssertionError("{} not reached".format(desc)) + + +@pytest.fixture +def lab(): + token = uuid.uuid4().hex[:5] + bridge = ("eff" + token)[:15] + base = str(TEST_ROOT / ("pglab-netem-effects-" + token)) + subnet = "10.239.{}.0/24".format(int(token[:2], 16) % 180 + 40) + + instance = PgLab( + pg_bin=PG_BIN, + base_dir=base, + subnet=subnet, + bridge_name=bridge, + os_user=OS_USER, + db_superuser=DB_USER, + ) + try: + yield instance + finally: + instance.close() + + +def node_ip(lab, hostno): + return str(list(lab.subnet.hosts())[hostno - 1]) + + +def make_pair(lab): + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init().start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + return primary, replica + + +def ping_times(node, target, count=4, timeout=10): + proc = run( + "ip", "netns", "exec", node.namespace, + "ping", "-n", "-c", str(count), "-W", "2", target, + check=False, + timeout=timeout, + ) + times = [ + float(x) + for x in re.findall(r"time[=<]([0-9.]+)\s*ms", proc.stdout) + ] + return proc, times + + +def qdisc_stats(node): + return run( + "ip", "netns", "exec", node.namespace, + "tc", "-s", "qdisc", "show", "dev", node.veth_ns, + ).stdout + + +def psql_query(node, sql, connect_timeout=2, timeout=6): + return run( + PSQL, + "-X", + "-h", node.ip, + "-p", str(node.port), + "-U", DB_USER, + "-d", "postgres", + "-Atq", + "-c", sql, + check=False, + timeout=timeout, + env={ + "PGCONNECT_TIMEOUT": str(connect_timeout), + "PGSSLMODE": "disable", + }, + ) + + +def test_netem_delay_changes_real_rtt(lab): + """Prove that netem changes packet RTT, not only qdisc configuration.""" + primary, replica = make_pair(lab) + + base_proc, base_times = ping_times(replica, primary.ip) + assert base_proc.returncode == 0 + assert len(base_times) >= 3 + baseline = sum(base_times) / len(base_times) + + replica.netem(delay_ms=200) + + delayed_proc, delayed_times = ping_times(replica, primary.ip) + assert delayed_proc.returncode == 0 + assert len(delayed_times) >= 3 + delayed = sum(delayed_times) / len(delayed_times) + + assert delayed >= baseline + 150, ( + "netem did not visibly increase RTT: baseline={:.3f} ms, " + "delayed={:.3f} ms".format(baseline, delayed) + ) + + replica.clear_netem() + + recovered_proc, recovered_times = ping_times(replica, primary.ip) + assert recovered_proc.returncode == 0 + recovered = sum(recovered_times) / len(recovered_times) + assert recovered < delayed - 100 + + +def test_netem_100_percent_loss_really_drops_packets(lab): + """100% loss must break connectivity and tc must account dropped packets.""" + primary, replica = make_pair(lab) + + ok, _ = ping_times(replica, primary.ip, count=2) + assert ok.returncode == 0 + + replica.netem(loss_pct=100) + + broken, times = ping_times(replica, primary.ip, count=3) + assert broken.returncode != 0 + assert times == [] + + stats = qdisc_stats(replica) + match = re.search(r"dropped\s+(\d+)", stats) + assert match is not None, stats + assert int(match.group(1)) > 0, stats + + replica.clear_netem() + + healed, times = ping_times(replica, primary.ip, count=2) + assert healed.returncode == 0 + assert len(times) == 2 + + +def test_netem_loss_breaks_real_postgresql_connection(lab): + """Show an application-visible failure: psql cannot connect through 100% loss.""" + _, replica = make_pair(lab) + + before = psql_query(replica, "select 42") + assert before.returncode == 0 + assert before.stdout.strip() == "42" + + replica.netem(loss_pct=100) + + started = time.monotonic() + broken = psql_query(replica, "select 42", connect_timeout=2, timeout=5) + elapsed = time.monotonic() - started + + assert broken.returncode != 0 + assert elapsed >= 1.0 + assert ( + "timeout" in broken.stderr.lower() + or "could not connect" in broken.stderr.lower() + or "connection" in broken.stderr.lower() + ), broken.stderr + + replica.clear_netem() + + after = psql_query(replica, "select 42") + assert after.returncode == 0 + assert after.stdout.strip() == "42" + + +def test_netem_rate_limit_is_observable_by_postgresql_client(lab): + """Rate limiting should turn a fast COPY into a visibly slow transfer.""" + _, replica = make_pair(lab) + + sql = ( + "copy (" + "select repeat('x', 1024) " + "from generate_series(1, 128)" + ") to stdout" + ) + + started = time.monotonic() + normal = run( + PSQL, + "-X", "-h", replica.ip, "-p", str(replica.port), + "-U", DB_USER, "-d", "postgres", "-q", "-c", sql, + check=False, + timeout=10, + env={"PGCONNECT_TIMEOUT": "2", "PGSSLMODE": "disable"}, + stdout=subprocess.DEVNULL, + ) + baseline = time.monotonic() - started + assert normal.returncode == 0, normal.stderr + + replica.netem(rate_kbit=256) + + started = time.monotonic() + limited = run( + PSQL, + "-X", "-h", replica.ip, "-p", str(replica.port), + "-U", DB_USER, "-d", "postgres", "-q", "-c", sql, + check=False, + timeout=15, + env={"PGCONNECT_TIMEOUT": "2", "PGSSLMODE": "disable"}, + stdout=subprocess.DEVNULL, + ) + throttled = time.monotonic() - started + assert limited.returncode == 0, limited.stderr + + assert throttled > 2.0, ( + "rate limit had no visible effect: baseline={:.3f}s throttled={:.3f}s" + .format(baseline, throttled) + ) + assert throttled > baseline * 3, ( + "rate limit effect too small: baseline={:.3f}s throttled={:.3f}s" + .format(baseline, throttled) + ) + + +def test_netem_loss_creates_real_replication_lag_then_recovers(lab): + """Prove that packet loss stalls streaming replication, then catchup heals it.""" + primary, replica = make_pair(lab) + + primary.execute("create table walhog(id int primary key, payload text)") + primary.execute("insert into walhog values (1, repeat('a', 1024))") + replica.catchup() + + baseline_lag = primary.execute(""" + select coalesce( + pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), + 0 + )::bigint + from pg_stat_replication + where application_name = 'r1' + """)[0][0] + assert baseline_lag < 1024 * 1024 + + replica.netem(loss_pct=100) + + primary.execute(""" + insert into walhog + select g, repeat(md5(g::text), 32) + from generate_series(2, 20002) as g + """) + primary.execute("select pg_switch_wal()") + time.sleep(2.0) + + lag = primary.execute(""" + select coalesce( + pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), + 0 + )::bigint + from pg_stat_replication + where application_name = 'r1' + """)[0][0] + + assert lag > 1024 * 1024, "expected replication lag, got {} bytes".format(lag) + + replica.clear_netem() + replica.catchup() + + assert replica.execute( + "select count(*), max(id) from walhog" + ) == [(20002, 20002)] + + recovered_lag = primary.execute(""" + select coalesce( + pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), + 0 + )::bigint + from pg_stat_replication + where application_name = 'r1' + """)[0][0] + + assert recovered_lag < lag + + +def test_replica_cannot_recover_after_required_wal_is_recycled(lab): + """A long enough outage must become unrecoverable without WAL retention. + + This deliberately runs without a physical replication slot and with + wal_keep_size=0. We first break replica egress with netem, wait until the + primary drops the walsender, then advance/checkpoint WAL until the segment + needed by the replica has actually disappeared from pg_wal. Restoring the + network must not magically heal the replica: a new base backup (or an + archive containing the missing WAL) is required. + """ + primary = lab.create_primary("p1", node_ip(lab, 10)) + primary.init() + + # Keep the test compact and deterministic: once the replica is disconnected, + # old WAL can be recycled aggressively. + primary.append_conf( + wal_keep_size=0, + min_wal_size="32MB", + max_wal_size="64MB", + wal_sender_timeout="2s", + checkpoint_timeout="1h", + ) + primary.start() + + replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) + replica.start() + replica.catchup() + + primary.execute("create table wal_gap(id int primary key)") + primary.execute("insert into wal_gap values (1)") + replica.catchup() + assert replica.execute("select array_agg(id order by id) from wal_gap") == [([1],)] + + # Capture the point at which both sides are known to be caught up. + disconnect_lsn = primary.execute( + "select pg_current_wal_lsn()::text" + )[0][0] + required_wal = primary.execute( + "select pg_walfile_name('{}'::pg_lsn)".format(disconnect_lsn) + )[0][0] + + assert primary.execute( + "select exists (select 1 from pg_ls_waldir() where name = '{}')".format( + required_wal + ) + ) == [(True,)] + + # 100% loss on replica egress drops ACKs/status replies. With the short + # wal_sender_timeout the primary closes the old streaming connection; all + # subsequent reconnect attempts from the replica are also dropped. + replica.netem(loss_pct=100) + + wait_until( + lambda: primary.execute(""" + select count(*) + from pg_stat_replication + where application_name = 'r1' + """)[0][0] == 0, + timeout=10, + desc="walsender to disappear after netem loss", + ) + + # No WAL is generated between the last catchup and the walsender going + # away. Now move sufficiently far ahead and checkpoint until the segment + # containing the replica's required position is really gone. We test the + # filesystem-visible WAL set rather than assuming that N switches are enough + # on every PostgreSQL version/configuration. + removed = False + switches = 0 + + for _ in range(8): + primary.execute( + "select pg_switch_wal() from generate_series(1, 4)" + ) + switches += 4 + primary.execute("checkpoint") + + exists = primary.execute( + "select exists (select 1 from pg_ls_waldir() where name = '{}')".format( + required_wal + ) + )[0][0] + + if not exists: + removed = True + break + + assert removed, ( + "required WAL {} was still present after {} forced WAL switches" + .format(required_wal, switches) + ) + + # Put an unmistakable change after the WAL gap. + primary.execute("insert into wal_gap values (2)") + target_lsn = primary.execute( + "select pg_current_wal_lsn()::text" + )[0][0] + + print( + "WAL gap created: disconnect_lsn={}, removed_wal={}, " + "switches={}, target_lsn={}".format( + disconnect_lsn, required_wal, switches, target_lsn + ) + ) + + # The network is healthy again, but the missing WAL is no longer available + # from the primary. Do NOT call Testgres catchup() here: its current + # implementation waits indefinitely (max_attempts=0) by design. + replica.clear_netem() + + # Give walreceiver several reconnect attempts. The replica itself is + # reachable from the controller again, so we can observe that replay does + # not cross the gap. + deadline = time.monotonic() + 8.0 + observed_counts = [] + + while time.monotonic() < deadline: + observed_counts.append( + replica.execute("select count(*) from wal_gap")[0][0] + ) + + caught_up = replica.execute( + "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn".format( + target_lsn + ) + )[0][0] + + assert not caught_up, ( + "replica unexpectedly crossed a WAL segment that was already " + "recycled on the primary" + ) + time.sleep(0.5) + + assert set(observed_counts) == {1}, observed_counts + assert primary.execute("select array_agg(id order by id) from wal_gap") == [([1, 2],)] + assert replica.execute("select array_agg(id order by id) from wal_gap") == [([1],)] + assert replica.execute("select pg_is_in_recovery()") == [(True,)] + + print( + "Replica stayed behind after network recovery as expected; " + "a new base backup or WAL archive is required." + ) From 901bced0417b193da7c000850e22b0d1600da837 Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Fri, 16 Oct 2026 18:37:31 +0700 Subject: [PATCH 06/11] Fix PgLab flake8 issues --- src/pglab.py | 1 - tests/test_pglab_faults.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/pglab.py b/src/pglab.py index 8ea3c798..1b9d8500 100644 --- a/src/pglab.py +++ b/src/pglab.py @@ -461,7 +461,6 @@ def __init__( self._firewall_source_added = False self._iptables_rule_added = False - hosts = self.subnet.hosts() default_bridge_ip = str(next(hosts)) self.bridge_ip = bridge_ip or default_bridge_ip diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py index ecbb2a63..677830bb 100644 --- a/tests/test_pglab_faults.py +++ b/tests/test_pglab_faults.py @@ -294,7 +294,6 @@ def test_cleanup_when_postgres_was_sigkilled(capsys): p = lab.create_primary("p1", node_ip(lab, 10)) p.init().start() - postmaster_pid = p.execute("select pg_postmaster_start_time() is not null, pg_backend_pid()")[0][1] # Find actual postmaster pid from postmaster.pid, then kill -9 it. pgdata = Path(p.pg.data_dir) From 2b4fbb186b3116231a4d1f31255adfe53b42533e Mon Sep 17 00:00:00 2001 From: Andrew Bille Date: Fri, 16 Oct 2026 18:39:36 +0700 Subject: [PATCH 07/11] Fix PgLab flake8 issues --- tests/test_pglab_faults.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py index 677830bb..d865d627 100644 --- a/tests/test_pglab_faults.py +++ b/tests/test_pglab_faults.py @@ -294,7 +294,6 @@ def test_cleanup_when_postgres_was_sigkilled(capsys): p = lab.create_primary("p1", node_ip(lab, 10)) p.init().start() - # Find actual postmaster pid from postmaster.pid, then kill -9 it. pgdata = Path(p.pg.data_dir) lines = (pgdata / "postmaster.pid").read_text().splitlines() From 93b8f029180af830b6320497b1e60b9bdf0697bd Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 19 Oct 2026 15:03:59 +0300 Subject: [PATCH 08/11] Revert "Fix PgLab flake8 issues" This reverts commit 2b4fbb186b3116231a4d1f31255adfe53b42533e. --- tests/test_pglab_faults.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py index d865d627..677830bb 100644 --- a/tests/test_pglab_faults.py +++ b/tests/test_pglab_faults.py @@ -294,6 +294,7 @@ def test_cleanup_when_postgres_was_sigkilled(capsys): p = lab.create_primary("p1", node_ip(lab, 10)) p.init().start() + # Find actual postmaster pid from postmaster.pid, then kill -9 it. pgdata = Path(p.pg.data_dir) lines = (pgdata / "postmaster.pid").read_text().splitlines() From 485f3ad74e65d7ec6935e47a09915b020222e029 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 19 Oct 2026 15:03:59 +0300 Subject: [PATCH 09/11] Revert "Fix PgLab flake8 issues" This reverts commit 901bced0417b193da7c000850e22b0d1600da837. --- src/pglab.py | 1 + tests/test_pglab_faults.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/pglab.py b/src/pglab.py index 1b9d8500..8ea3c798 100644 --- a/src/pglab.py +++ b/src/pglab.py @@ -461,6 +461,7 @@ def __init__( self._firewall_source_added = False self._iptables_rule_added = False + hosts = self.subnet.hosts() default_bridge_ip = str(next(hosts)) self.bridge_ip = bridge_ip or default_bridge_ip diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py index 677830bb..ecbb2a63 100644 --- a/tests/test_pglab_faults.py +++ b/tests/test_pglab_faults.py @@ -294,6 +294,7 @@ def test_cleanup_when_postgres_was_sigkilled(capsys): p = lab.create_primary("p1", node_ip(lab, 10)) p.init().start() + postmaster_pid = p.execute("select pg_postmaster_start_time() is not null, pg_backend_pid()")[0][1] # Find actual postmaster pid from postmaster.pid, then kill -9 it. pgdata = Path(p.pg.data_dir) From 9bb24b22a2b43131d86d4171e6c9fe73525997ad Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 19 Oct 2026 15:03:59 +0300 Subject: [PATCH 10/11] Revert "Run PgLab integration tests in CI" This reverts commit 29d8f98093d00054b3998f090d54633d01ac950b. --- .github/workflows/ci.yml | 70 ----- tests/helpers/pglab_test_config.py | 25 -- tests/test_pglab.py | 245 --------------- tests/test_pglab_compat.py | 29 -- tests/test_pglab_faults.py | 393 ------------------------ tests/test_pglab_netem.py | 306 ------------------- tests/test_pglab_netem_effects.py | 469 ----------------------------- 7 files changed, 1537 deletions(-) delete mode 100644 tests/helpers/pglab_test_config.py delete mode 100644 tests/test_pglab.py delete mode 100644 tests/test_pglab_compat.py delete mode 100644 tests/test_pglab_faults.py delete mode 100644 tests/test_pglab_netem.py delete mode 100644 tests/test_pglab_netem_effects.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84357173..2121fe92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,76 +289,6 @@ jobs: name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} path: "${{ env.RUN_CFG__LOGS_DIR }}/" - test-pglab: - runs-on: ubuntu-24.04 - needs: build-check - name: "test: pglab | ubuntu-24.04 | pg16" - - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Set up Python 3.12 - uses: actions/setup-python@v7 - with: - python-version: "3.12" - cache: 'pip' - - - name: Install PostgreSQL and PgLab system dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - postgresql-16 \ - postgresql-client-16 \ - libpq-dev \ - iproute2 \ - iputils-ping \ - iptables \ - util-linux - - - name: Install Python test dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r tests/requirements.txt - - - name: PgLab preflight - run: | - set -eux - "$(pg_config --bindir)/postgres" --version - pg_config --bindir - findmnt -n -o SOURCE,FSTYPE -T /var/tmp - mount | grep cgroup || true - cat /sys/fs/cgroup/cgroup.controllers - cat /sys/fs/cgroup/cgroup.subtree_control || true - - - name: Run privileged PgLab tests - run: | - set -eux - mkdir -p "${{ github.workspace }}/logs-pglab" - PYTHON="$(command -v python)" - PG_BIN="$(pg_config --bindir)" - sudo env \ - PATH="$PATH" \ - TESTGRES_PGLAB_TESTS=1 \ - PG_BIN="$PG_BIN" \ - PGLAB_OS_USER="$USER" \ - PGLAB_DB_USER=postgres \ - PGLAB_TEST_ROOT=/var/tmp \ - TEST_CFG__LOG_DIR="${{ github.workspace }}/logs-pglab" \ - "$PYTHON" -m pytest -vv -s \ - tests/test_pglab_compat.py \ - tests/test_pglab.py \ - tests/test_pglab_faults.py \ - tests/test_pglab_netem.py \ - tests/test_pglab_netem_effects.py - - - name: Upload PgLab logs - uses: actions/upload-artifact@v7 - if: always() - with: - name: testgres--pglab-logs--id${{ github.run_id }} - path: logs-pglab/ - test-macos-14: runs-on: macos-14 needs: build-check diff --git a/tests/helpers/pglab_test_config.py b/tests/helpers/pglab_test_config.py deleted file mode 100644 index 0e7add01..00000000 --- a/tests/helpers/pglab_test_config.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -from pathlib import Path - -import pytest - - -PG_BIN = os.environ.get("PG_BIN") -OS_USER = os.environ.get("PGLAB_OS_USER", os.environ.get("SUDO_USER", "postgres")) -DB_USER = os.environ.get("PGLAB_DB_USER", "postgres") -TEST_ROOT = Path(os.environ.get("PGLAB_TEST_ROOT", "/var/tmp")) - -PGLAB_INTEGRATION_MARKS = [ - pytest.mark.skipif( - os.environ.get("TESTGRES_PGLAB_TESTS") != "1", - reason="set TESTGRES_PGLAB_TESTS=1 to run privileged PgLab tests", - ), - pytest.mark.skipif( - os.geteuid() != 0, - reason="PgLab integration tests require root", - ), - pytest.mark.skipif( - not PG_BIN, - reason="PG_BIN must point to the PostgreSQL bin directory", - ), -] diff --git a/tests/test_pglab.py b/tests/test_pglab.py deleted file mode 100644 index 514301ff..00000000 --- a/tests/test_pglab.py +++ /dev/null @@ -1,245 +0,0 @@ -import shutil -import subprocess -import uuid -from pathlib import Path - -import pytest - -from src.pglab import PgLab, IOLimits -from .helpers.pglab_test_config import ( - DB_USER, - OS_USER, - PG_BIN, - PGLAB_INTEGRATION_MARKS, - TEST_ROOT, -) - - -pytestmark = PGLAB_INTEGRATION_MARKS - - -def run(*args, check=True): - return subprocess.run( - [str(a) for a in args], - check=check, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - -def netns_exists(name): - r = run("ip", "netns", "list") - return any(line.split()[0] == name for line in r.stdout.splitlines() if line.strip()) - - -def firewall_source_zone(subnet): - fw = shutil.which("firewall-cmd") - if fw is None: - return None - if run(fw, "--state", check=False).returncode != 0: - return None - r = run(fw, "--get-zone-of-source={}".format(subnet), check=False) - if r.returncode != 0: - return "" - return r.stdout.strip() - - -@pytest.fixture -def lab(): - # Keep names short: Linux IFNAMSIZ is 16 bytes including NUL. - token = uuid.uuid4().hex[:5] - bridge = "plt" + token - base = str(TEST_ROOT / ("pglab-pytest-" + token)) - - instance = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet="10.231.{}.0/24".format(int(token[:2], 16) % 200 + 20), - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - try: - yield instance - finally: - instance.close() - shutil.rmtree(base, ignore_errors=True) - - -def test_primary_starts_in_netns_and_uses_tcp(lab): - primary = lab.create_primary("p1", "10.231.{}.10".format(lab.subnet.network_address.packed[2])) - primary.init().start() - - assert primary.execute("select 1") == [(1,)] - expected = primary.ip - assert primary.execute( - "select host(inet_server_addr()) = '{}'".format(expected) - ) == [(True,)] - - # We deliberately disable the shared host Unix socket, because netns does - # not isolate the filesystem and every node may use TCP port 5432. - assert primary.execute("show unix_socket_directories") == [("",)] - - -def test_replication_and_catchup(lab): - third = lab.subnet.network_address.packed[2] - primary = lab.create_primary("p1", "10.231.{}.10".format(third)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", "10.231.{}.11".format(third)) - replica.start() - replica.catchup() - - assert replica.execute("select pg_is_in_recovery()") == [(True,)] - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1), (2), (3)") - replica.catchup() - - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] - - rows = primary.execute(""" - select application_name, state, host(client_addr) - from pg_stat_replication - where application_name = 'r1' - """) - assert rows == [("r1", "streaming", replica.ip)] - - -def test_replica_target_port_and_storage_root(lab): - third = lab.subnet.network_address.packed[2] - primary = lab.create_primary("p1", "10.231.{}.10".format(third), port=5432) - primary.init().start() - - replica = lab.create_replica( - "p1", - "r1", - "10.231.{}.11".format(third), - port=5433, - ) - - assert replica.port == 5433 - assert Path(replica.pg.base_dir) == lab.base_dir / "r1" - - replica.start() - replica.catchup() - - assert replica.execute("show port") == [("5433",)] - assert replica.execute("select pg_is_in_recovery()") == [(True,)] - - -def test_partition_and_heal(lab): - third = lab.subnet.network_address.packed[2] - primary = lab.create_primary("p1", "10.231.{}.10".format(third)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", "10.231.{}.11".format(third)) - replica.start() - replica.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - assert replica.execute("select count(*) from t") == [(1,)] - - lab.partition("p1", "r1") - primary.execute("insert into t values (2)") - - # Controller -> replica connectivity remains available; only node-to-node - # traffic is blackholed. - assert replica.execute("select count(*) from t") == [(1,)] - - # The sender should stop being streaming after the partition has had a - # chance to break the connection. Don't make timing the essence of this - # test; the important assertion is that replay cannot advance. - lab.heal_partition("p1", "r1") - replica.catchup() - - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - -def test_postgres_backend_is_in_node_cgroup(lab): - third = lab.subnet.network_address.packed[2] - primary = lab.create_primary( - "p1", - "10.231.{}.10".format(third), - io=IOLimits(read_bps=50 * 1024 * 1024), - ) - primary.init().start() - - backend_pid = primary.execute("select pg_backend_pid()")[0][0] - membership = Path(primary.os_ops.cgroup_membership_file) - - assert membership.exists() - - pids = { - int(line) - for line in membership.read_text().splitlines() - if line.strip().isdigit() - } - assert backend_pid in pids - - -def test_io_limit_is_programmed(lab): - third = lab.subnet.network_address.packed[2] - limit = 32 * 1024 * 1024 - - primary = lab.create_primary( - "p1", - "10.231.{}.10".format(third), - io=IOLimits(read_bps=limit), - ) - primary.init().start() - - cg = Path(primary.cgroup) - version = lab.cgroup_backend.version - - if version == 2: - text = (cg / "io.max").read_text() - assert "rbps={}".format(limit) in text - elif version == 1: - text = (cg / "blkio.throttle.read_bps_device").read_text() - assert any( - line.split()[-1] == str(limit) - for line in text.splitlines() - if line.strip() - ) - else: - pytest.fail("unsupported cgroup backend version: {!r}".format(version)) - - -def test_cleanup_removes_namespace_bridge_and_runtime_firewall(): - token = uuid.uuid4().hex[:5] - bridge = "plc" + token - base = str(TEST_ROOT / ("pglab-cleanup-" + token)) - subnet = "10.232.{}.0/24".format(int(token[:2], 16) % 200 + 20) - - zone_before = firewall_source_zone(subnet) - - lab = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - - node = lab.create_primary("cleanup", str(list(lab.subnet.hosts())[9])) - node.init().start() - namespace = node.namespace - - assert netns_exists(namespace) - assert run("ip", "link", "show", "dev", bridge, check=False).returncode == 0 - - lab.close() - - assert not netns_exists(namespace) - assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 - assert not Path(base).exists() - - # PgLab must not remove a pre-existing user firewalld assignment. If there - # was no assignment before, there must not be one left afterwards. - if zone_before == "": - assert firewall_source_zone(subnet) == "" diff --git a/tests/test_pglab_compat.py b/tests/test_pglab_compat.py deleted file mode 100644 index 23862692..00000000 --- a/tests/test_pglab_compat.py +++ /dev/null @@ -1,29 +0,0 @@ -import inspect - -from src.pglab import NamespaceOperations -from src.backup import NodeBackup -from src.node import PostgresNode - - -def test_testgres_target_execution_context_api(): - replicate = inspect.signature(PostgresNode.replicate).parameters - assert "target_os_ops" in replicate - assert "target_host" in replicate - assert "target_port" in replicate - - backup_init = inspect.signature(NodeBackup.__init__).parameters - assert "os_ops" in backup_init - - spawn_replica = inspect.signature(NodeBackup.spawn_replica).parameters - assert "target_host" in spawn_replica - assert "target_port" in spawn_replica - - -def test_namespace_operations_exposes_explicit_pg_bin(): - ops = NamespaceOperations( - namespace="pglab-test-unused", - os_user="postgres", - db_user="postgres", - pg_bin="/opt/postgresql/bin", - ) - assert ops.environ("PG_BIN") == "/opt/postgresql/bin" diff --git a/tests/test_pglab_faults.py b/tests/test_pglab_faults.py deleted file mode 100644 index ecbb2a63..00000000 --- a/tests/test_pglab_faults.py +++ /dev/null @@ -1,393 +0,0 @@ -import os -import shutil -import subprocess -import time -import uuid -from pathlib import Path - -import pytest - -from src.pglab import PgLab, IOLimits -from .helpers.pglab_test_config import ( - DB_USER, - OS_USER, - PG_BIN, - PGLAB_INTEGRATION_MARKS, - TEST_ROOT, -) - - -pytestmark = PGLAB_INTEGRATION_MARKS - - -def run(*args, check=True): - return subprocess.run( - [str(a) for a in args], - check=check, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - -def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): - deadline = time.monotonic() + timeout - last = None - while time.monotonic() < deadline: - try: - if fn(): - return - except Exception as exc: - last = exc - time.sleep(interval) - if last is not None: - raise AssertionError("{} not reached; last error: {!r}".format(desc, last)) - raise AssertionError("{} not reached".format(desc)) - - -def make_lab(prefix="flt", subnet_octet=None): - token = uuid.uuid4().hex[:5] - bridge = (prefix + token)[:15] - base = str(TEST_ROOT / ("pglab-faults-" + token)) - if subnet_octet is None: - subnet_octet = int(token[:2], 16) % 180 + 40 - subnet = "10.233.{}.0/24".format(subnet_octet) - - return PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - - -@pytest.fixture -def lab(): - instance = make_lab() - try: - yield instance - finally: - instance.close() - - -def node_ip(lab, hostno): - return str(list(lab.subnet.hosts())[hostno - 1]) - - -def pg_stat_replication(primary): - return primary.execute(""" - select application_name, state, host(client_addr) - from pg_stat_replication - order by application_name - """) - - -def test_two_replicas_same_port_and_independent_catchup(lab): - primary = lab.create_primary("p1", node_ip(lab, 10), port=5432) - primary.init().start() - - r1 = lab.create_replica("p1", "r1", node_ip(lab, 11), port=5432) - r2 = lab.create_replica("p1", "r2", node_ip(lab, 12), port=5432) - - r1.start() - r2.start() - r1.catchup() - r2.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1), (2), (3)") - r1.catchup() - r2.catchup() - - assert r1.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] - assert r2.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] - - rows = pg_stat_replication(primary) - assert rows == [ - ("r1", "streaming", r1.ip), - ("r2", "streaming", r2.ip), - ] - - assert primary.port == r1.port == r2.port == 5432 - - -def test_replica_restart_rejoins_streaming(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - replica.stop() - primary.execute("insert into t values (2), (3)") - - replica.start() - replica.catchup() - - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] - - wait_until( - lambda: pg_stat_replication(primary) == [ - ("r1", "streaming", replica.ip) - ], - desc="replica streaming after restart", - ) - - -def test_primary_restart_replica_recovers(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - primary.stop() - - # Replica remains queryable in recovery while upstream is unavailable. - assert replica.execute("select pg_is_in_recovery()") == [(True,)] - assert replica.execute("select array_agg(i order by i) from t") == [([1],)] - - primary.start() - primary.execute("insert into t values (2)") - replica.catchup() - - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - -def test_network_down_up_recovers_streaming(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - replica.network_down() - primary.execute("insert into t values (2)") - - # Controller may lose direct reachability to the replica while eth0 is down, - # so do not query it here. Restore networking first. - replica.network_up() - - wait_until( - lambda: run( - "ip", "netns", "exec", replica.namespace, - "ping", "-c", "1", "-W", "1", primary.ip, - check=False, - ).returncode == 0, - timeout=10, - desc="replica network restored", - ) - - replica.catchup() - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - -def test_live_io_limit_change(lab): - first = 64 * 1024 * 1024 - second = 8 * 1024 * 1024 - - primary = lab.create_primary( - "p1", - node_ip(lab, 10), - io=IOLimits(read_bps=first), - ) - primary.init().start() - - lab.set_io_limits("p1", IOLimits(read_bps=second)) - - cg = Path(primary.cgroup) - if lab.cgroup_backend.version == 2: - text = (cg / "io.max").read_text() - assert "rbps={}".format(second) in text - assert "rbps={}".format(first) not in text - elif lab.cgroup_backend.version == 1: - text = (cg / "blkio.throttle.read_bps_device").read_text() - values = [ - line.split()[-1] - for line in text.splitlines() - if line.strip() - ] - assert str(second) in values - else: - pytest.fail("unsupported cgroup backend") - - -def test_exception_inside_context_cleans_everything(): - token = uuid.uuid4().hex[:5] - bridge = ("boom" + token)[:15] - base = str(TEST_ROOT / ("pglab-boom-" + token)) - subnet = "10.234.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - namespace = None - with pytest.raises(RuntimeError, match="boom"): - with PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) as lab: - p = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) - namespace = p.namespace - p.init().start() - raise RuntimeError("boom") - - assert namespace is not None - assert namespace not in run("ip", "netns", "list").stdout - assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 - assert not Path(base).exists() - - -def test_keep_data_preserves_pgdata(): - token = uuid.uuid4().hex[:5] - bridge = ("keep" + token)[:15] - base = TEST_ROOT / ("pglab-keep-" + token) - subnet = "10.236.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - lab = PgLab( - pg_bin=PG_BIN, - base_dir=str(base), - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - keep_data=True, - ) - - primary = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) - primary.init().start() - primary.execute("create table keep_me(i int)") - - pgdata = Path(primary.pg.data_dir) - lab.close() - - try: - assert pgdata.is_dir() - assert (pgdata / "PG_VERSION").is_file() - finally: - shutil.rmtree(base, ignore_errors=True) - - -def test_cleanup_when_postgres_was_sigkilled(capsys): - lab = make_lab(prefix="kill") - bridge = lab.bridge_name - base = lab.base_dir - - p = lab.create_primary("p1", node_ip(lab, 10)) - p.init().start() - - postmaster_pid = p.execute("select pg_postmaster_start_time() is not null, pg_backend_pid()")[0][1] - - # Find actual postmaster pid from postmaster.pid, then kill -9 it. - pgdata = Path(p.pg.data_dir) - lines = (pgdata / "postmaster.pid").read_text().splitlines() - actual_postmaster = int(lines[0]) - - os.kill(actual_postmaster, 9) - - wait_until( - lambda: not Path("/proc/{}".format(actual_postmaster)).exists(), - timeout=10, - desc="postmaster SIGKILL", - ) - - # close() must tolerate the already-dead server and still clean isolation. - lab.close() - - captured = capsys.readouterr() - assert "cannot stop node" not in captured.err - assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 - assert not Path(base).exists() - - -def test_recreate_same_lab_after_close(): - token = uuid.uuid4().hex[:5] - bridge = ("same" + token)[:15] - base = str(TEST_ROOT / ("pglab-same-" + token)) - subnet = "10.235.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - for cycle in (1, 2): - with PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) as lab: - p = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) - p.init().start() - assert p.execute("select {}".format(cycle)) == [(cycle,)] - - assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 - assert not Path(base).exists() - - -def test_partition_only_one_replica(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - r1 = lab.create_replica("p1", "r1", node_ip(lab, 11)) - r2 = lab.create_replica("p1", "r2", node_ip(lab, 12)) - r1.start() - r2.start() - r1.catchup() - r2.catchup() - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - r1.catchup() - r2.catchup() - - lab.partition("p1", "r1") - primary.execute("insert into t values (2)") - - # r2 should continue normally while r1 is isolated. - r2.catchup() - assert r2.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - # r1 is still queryable from the controller and should not have replayed 2. - assert r1.execute("select array_agg(i order by i) from t") == [([1],)] - - lab.heal_partition("p1", "r1") - r1.catchup() - assert r1.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - -def test_replication_survives_repeated_partition_cycles(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - - primary.execute("create table t(i int primary key)") - - for i in range(1, 6): - lab.partition("p1", "r1") - primary.execute("insert into t values ({})".format(i)) - lab.heal_partition("p1", "r1") - replica.catchup() - - assert replica.execute( - "select coalesce(max(i), 0), count(*) from t" - ) == [(i, i)] diff --git a/tests/test_pglab_netem.py b/tests/test_pglab_netem.py deleted file mode 100644 index 6d905593..00000000 --- a/tests/test_pglab_netem.py +++ /dev/null @@ -1,306 +0,0 @@ -import subprocess -import time -import uuid -from pathlib import Path - -import pytest - -from src.pglab import PgLab, PgLabError -from .helpers.pglab_test_config import ( - DB_USER, - OS_USER, - PG_BIN, - PGLAB_INTEGRATION_MARKS, - TEST_ROOT, -) - - -pytestmark = PGLAB_INTEGRATION_MARKS - - -def run(*args, check=True): - return subprocess.run( - [str(a) for a in args], - check=check, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - -def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): - deadline = time.monotonic() + timeout - last = None - while time.monotonic() < deadline: - try: - if fn(): - return - except Exception as exc: - last = exc - time.sleep(interval) - if last is not None: - raise AssertionError("{} not reached; last error: {!r}".format(desc, last)) - raise AssertionError("{} not reached".format(desc)) - - -@pytest.fixture -def lab(): - token = uuid.uuid4().hex[:5] - bridge = ("net" + token)[:15] - base = str(TEST_ROOT / ("pglab-netem-" + token)) - subnet = "10.236.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - instance = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - try: - yield instance - finally: - instance.close() - - -def node_ip(lab, hostno): - return str(list(lab.subnet.hosts())[hostno - 1]) - - -def make_pair(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - return primary, replica - - -def qdisc(node): - return run( - "ip", "netns", "exec", node.namespace, - "tc", "qdisc", "show", "dev", node.veth_ns, - ).stdout - - -def postmaster_pid(node): - return int((Path(node.pg.data_dir) / "postmaster.pid").read_text().splitlines()[0]) - - -def proc_state(pid): - for line in Path("/proc/{}/status".format(pid)).read_text().splitlines(): - if line.startswith("State:"): - return line.split()[1] - raise AssertionError("no State in /proc/{}/status".format(pid)) - - -def test_netem_delay_is_installed_and_cleared(lab): - _, replica = make_pair(lab) - - replica.netem(delay_ms=150, jitter_ms=20) - - text = qdisc(replica) - assert "netem" in text - assert "delay" in text - assert "150" in text - - replica.clear_netem() - assert "netem" not in qdisc(replica) - - -def test_replication_survives_latency_and_loss(lab): - primary, replica = make_pair(lab) - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - # Large enough to exercise retransmits without making the test excessively - # slow. The fault applies to replica egress, including TCP ACKs. - replica.netem(delay_ms=80, jitter_ms=20, loss_pct=2) - - primary.execute( - "insert into t select i from generate_series(2, 1001) as g(i)" - ) - - # Avoid an aggressive Testgres catchup timeout while netem is active. - time.sleep(1.0) - replica.clear_netem() - replica.catchup() - - assert replica.execute("select count(*), max(i) from t") == [(1001, 1001)] - - -def test_netem_total_loss_then_recovery(lab): - primary, replica = make_pair(lab) - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - replica.netem(loss_pct=100) - primary.execute("insert into t values (2), (3)") - - assert "loss 100%" in qdisc(replica) - - replica.clear_netem() - replica.catchup() - - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3],)] - - -def test_netem_can_be_replaced_live(lab): - _, replica = make_pair(lab) - - replica.netem(delay_ms=50) - first = qdisc(replica) - assert "50" in first - - replica.netem(delay_ms=250, loss_pct=1) - second = qdisc(replica) - assert "250" in second - assert "loss 1%" in second - - replica.clear_netem() - - -def test_netem_validation(lab, capsys): - node = lab.create_primary("p1", node_ip(lab, 10)) - - with pytest.raises(PgLabError): - node.netem() - - with pytest.raises(PgLabError): - node.netem(loss_pct=101) - - with pytest.raises(PgLabError): - node.netem(jitter_ms=10) - - with pytest.raises(PgLabError): - node.netem(reorder_pct=10) - - # Closing an intentionally uninitialized node should not leak Testgres' - # internal stop diagnostic to the test output. - lab.close() - captured = capsys.readouterr() - assert "cannot stop node" not in captured.err - - -def test_freeze_replica_then_resume_and_catchup(lab): - primary, replica = make_pair(lab) - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - pid = postmaster_pid(replica) - replica.freeze() - - wait_until( - lambda: proc_state(pid) in ("T", "t"), - timeout=5, - desc="replica postmaster stopped", - ) - - primary.execute("insert into t values (2), (3), (4)") - - replica.unfreeze() - - wait_until( - lambda: proc_state(pid) not in ("T", "t"), - timeout=5, - desc="replica postmaster resumed", - ) - - replica.catchup() - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2, 3, 4],)] - - -def test_freeze_primary_then_resume(lab): - primary, replica = make_pair(lab) - - primary.execute("create table t(i int primary key)") - primary.execute("insert into t values (1)") - replica.catchup() - - pid = postmaster_pid(primary) - primary.freeze() - - wait_until( - lambda: proc_state(pid) in ("T", "t"), - timeout=5, - desc="primary postmaster stopped", - ) - - # Replica remains readable while upstream is frozen. - assert replica.execute("select array_agg(i order by i) from t") == [([1],)] - - primary.unfreeze() - wait_until( - lambda: proc_state(pid) not in ("T", "t"), - timeout=5, - desc="primary postmaster resumed", - ) - - primary.execute("insert into t values (2)") - replica.catchup() - assert replica.execute("select array_agg(i order by i) from t") == [([1, 2],)] - - -def test_close_unfreezes_before_cleanup(): - token = uuid.uuid4().hex[:5] - bridge = ("frz" + token)[:15] - base = str(TEST_ROOT / ("pglab-frozen-close-" + token)) - subnet = "10.237.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - lab = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - - node = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) - node.init().start() - node.freeze() - - # Regression test: close() must not hang trying to pg_ctl-stop SIGSTOPed PG. - started = time.monotonic() - lab.close() - elapsed = time.monotonic() - started - - assert elapsed < 15 - assert run("ip", "link", "show", "dev", bridge, check=False).returncode != 0 - assert not Path(base).exists() - - -def test_close_cleans_netem(): - token = uuid.uuid4().hex[:5] - bridge = ("tc" + token)[:15] - base = str(TEST_ROOT / ("pglab-netem-close-" + token)) - subnet = "10.238.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - lab = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - - node = lab.create_primary("p1", str(list(lab.subnet.hosts())[9])) - node.init().start() - node.netem(delay_ms=100, loss_pct=1) - assert "netem" in qdisc(node) - - lab.close() - - # Namespace deletion itself removes the qdisc; the important thing is that - # cleanup succeeds while a netem profile is active. - assert node.namespace not in run("ip", "netns", "list").stdout - assert not Path(base).exists() diff --git a/tests/test_pglab_netem_effects.py b/tests/test_pglab_netem_effects.py deleted file mode 100644 index 7215b630..00000000 --- a/tests/test_pglab_netem_effects.py +++ /dev/null @@ -1,469 +0,0 @@ -import os -import re -import shutil -import subprocess -import time -import uuid -from pathlib import Path - -import pytest - -from src.pglab import PgLab -from .helpers.pglab_test_config import ( - DB_USER, - OS_USER, - PG_BIN, - PGLAB_INTEGRATION_MARKS, - TEST_ROOT, -) - - -PSQL = str(Path(PG_BIN) / "psql") if PG_BIN else None - -pytestmark = PGLAB_INTEGRATION_MARKS + [ - pytest.mark.skipif(shutil.which("tc") is None, reason="tc/iproute2 is required"), -] - - -def run(*args, check=True, timeout=None, env=None, stdout=subprocess.PIPE): - full_env = os.environ.copy() - full_env["LC_ALL"] = "C" - if env: - full_env.update(env) - - return subprocess.run( - [str(a) for a in args], - check=check, - text=True, - stdout=stdout, - stderr=subprocess.PIPE, - timeout=timeout, - env=full_env, - ) - - -def wait_until(fn, timeout=15.0, interval=0.2, desc="condition"): - deadline = time.monotonic() + timeout - last = None - while time.monotonic() < deadline: - try: - if fn(): - return - except Exception as exc: - last = exc - time.sleep(interval) - - if last is not None: - raise AssertionError( - "{} not reached; last error: {!r}".format(desc, last) - ) - raise AssertionError("{} not reached".format(desc)) - - -@pytest.fixture -def lab(): - token = uuid.uuid4().hex[:5] - bridge = ("eff" + token)[:15] - base = str(TEST_ROOT / ("pglab-netem-effects-" + token)) - subnet = "10.239.{}.0/24".format(int(token[:2], 16) % 180 + 40) - - instance = PgLab( - pg_bin=PG_BIN, - base_dir=base, - subnet=subnet, - bridge_name=bridge, - os_user=OS_USER, - db_superuser=DB_USER, - ) - try: - yield instance - finally: - instance.close() - - -def node_ip(lab, hostno): - return str(list(lab.subnet.hosts())[hostno - 1]) - - -def make_pair(lab): - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init().start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - return primary, replica - - -def ping_times(node, target, count=4, timeout=10): - proc = run( - "ip", "netns", "exec", node.namespace, - "ping", "-n", "-c", str(count), "-W", "2", target, - check=False, - timeout=timeout, - ) - times = [ - float(x) - for x in re.findall(r"time[=<]([0-9.]+)\s*ms", proc.stdout) - ] - return proc, times - - -def qdisc_stats(node): - return run( - "ip", "netns", "exec", node.namespace, - "tc", "-s", "qdisc", "show", "dev", node.veth_ns, - ).stdout - - -def psql_query(node, sql, connect_timeout=2, timeout=6): - return run( - PSQL, - "-X", - "-h", node.ip, - "-p", str(node.port), - "-U", DB_USER, - "-d", "postgres", - "-Atq", - "-c", sql, - check=False, - timeout=timeout, - env={ - "PGCONNECT_TIMEOUT": str(connect_timeout), - "PGSSLMODE": "disable", - }, - ) - - -def test_netem_delay_changes_real_rtt(lab): - """Prove that netem changes packet RTT, not only qdisc configuration.""" - primary, replica = make_pair(lab) - - base_proc, base_times = ping_times(replica, primary.ip) - assert base_proc.returncode == 0 - assert len(base_times) >= 3 - baseline = sum(base_times) / len(base_times) - - replica.netem(delay_ms=200) - - delayed_proc, delayed_times = ping_times(replica, primary.ip) - assert delayed_proc.returncode == 0 - assert len(delayed_times) >= 3 - delayed = sum(delayed_times) / len(delayed_times) - - assert delayed >= baseline + 150, ( - "netem did not visibly increase RTT: baseline={:.3f} ms, " - "delayed={:.3f} ms".format(baseline, delayed) - ) - - replica.clear_netem() - - recovered_proc, recovered_times = ping_times(replica, primary.ip) - assert recovered_proc.returncode == 0 - recovered = sum(recovered_times) / len(recovered_times) - assert recovered < delayed - 100 - - -def test_netem_100_percent_loss_really_drops_packets(lab): - """100% loss must break connectivity and tc must account dropped packets.""" - primary, replica = make_pair(lab) - - ok, _ = ping_times(replica, primary.ip, count=2) - assert ok.returncode == 0 - - replica.netem(loss_pct=100) - - broken, times = ping_times(replica, primary.ip, count=3) - assert broken.returncode != 0 - assert times == [] - - stats = qdisc_stats(replica) - match = re.search(r"dropped\s+(\d+)", stats) - assert match is not None, stats - assert int(match.group(1)) > 0, stats - - replica.clear_netem() - - healed, times = ping_times(replica, primary.ip, count=2) - assert healed.returncode == 0 - assert len(times) == 2 - - -def test_netem_loss_breaks_real_postgresql_connection(lab): - """Show an application-visible failure: psql cannot connect through 100% loss.""" - _, replica = make_pair(lab) - - before = psql_query(replica, "select 42") - assert before.returncode == 0 - assert before.stdout.strip() == "42" - - replica.netem(loss_pct=100) - - started = time.monotonic() - broken = psql_query(replica, "select 42", connect_timeout=2, timeout=5) - elapsed = time.monotonic() - started - - assert broken.returncode != 0 - assert elapsed >= 1.0 - assert ( - "timeout" in broken.stderr.lower() - or "could not connect" in broken.stderr.lower() - or "connection" in broken.stderr.lower() - ), broken.stderr - - replica.clear_netem() - - after = psql_query(replica, "select 42") - assert after.returncode == 0 - assert after.stdout.strip() == "42" - - -def test_netem_rate_limit_is_observable_by_postgresql_client(lab): - """Rate limiting should turn a fast COPY into a visibly slow transfer.""" - _, replica = make_pair(lab) - - sql = ( - "copy (" - "select repeat('x', 1024) " - "from generate_series(1, 128)" - ") to stdout" - ) - - started = time.monotonic() - normal = run( - PSQL, - "-X", "-h", replica.ip, "-p", str(replica.port), - "-U", DB_USER, "-d", "postgres", "-q", "-c", sql, - check=False, - timeout=10, - env={"PGCONNECT_TIMEOUT": "2", "PGSSLMODE": "disable"}, - stdout=subprocess.DEVNULL, - ) - baseline = time.monotonic() - started - assert normal.returncode == 0, normal.stderr - - replica.netem(rate_kbit=256) - - started = time.monotonic() - limited = run( - PSQL, - "-X", "-h", replica.ip, "-p", str(replica.port), - "-U", DB_USER, "-d", "postgres", "-q", "-c", sql, - check=False, - timeout=15, - env={"PGCONNECT_TIMEOUT": "2", "PGSSLMODE": "disable"}, - stdout=subprocess.DEVNULL, - ) - throttled = time.monotonic() - started - assert limited.returncode == 0, limited.stderr - - assert throttled > 2.0, ( - "rate limit had no visible effect: baseline={:.3f}s throttled={:.3f}s" - .format(baseline, throttled) - ) - assert throttled > baseline * 3, ( - "rate limit effect too small: baseline={:.3f}s throttled={:.3f}s" - .format(baseline, throttled) - ) - - -def test_netem_loss_creates_real_replication_lag_then_recovers(lab): - """Prove that packet loss stalls streaming replication, then catchup heals it.""" - primary, replica = make_pair(lab) - - primary.execute("create table walhog(id int primary key, payload text)") - primary.execute("insert into walhog values (1, repeat('a', 1024))") - replica.catchup() - - baseline_lag = primary.execute(""" - select coalesce( - pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), - 0 - )::bigint - from pg_stat_replication - where application_name = 'r1' - """)[0][0] - assert baseline_lag < 1024 * 1024 - - replica.netem(loss_pct=100) - - primary.execute(""" - insert into walhog - select g, repeat(md5(g::text), 32) - from generate_series(2, 20002) as g - """) - primary.execute("select pg_switch_wal()") - time.sleep(2.0) - - lag = primary.execute(""" - select coalesce( - pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), - 0 - )::bigint - from pg_stat_replication - where application_name = 'r1' - """)[0][0] - - assert lag > 1024 * 1024, "expected replication lag, got {} bytes".format(lag) - - replica.clear_netem() - replica.catchup() - - assert replica.execute( - "select count(*), max(id) from walhog" - ) == [(20002, 20002)] - - recovered_lag = primary.execute(""" - select coalesce( - pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn), - 0 - )::bigint - from pg_stat_replication - where application_name = 'r1' - """)[0][0] - - assert recovered_lag < lag - - -def test_replica_cannot_recover_after_required_wal_is_recycled(lab): - """A long enough outage must become unrecoverable without WAL retention. - - This deliberately runs without a physical replication slot and with - wal_keep_size=0. We first break replica egress with netem, wait until the - primary drops the walsender, then advance/checkpoint WAL until the segment - needed by the replica has actually disappeared from pg_wal. Restoring the - network must not magically heal the replica: a new base backup (or an - archive containing the missing WAL) is required. - """ - primary = lab.create_primary("p1", node_ip(lab, 10)) - primary.init() - - # Keep the test compact and deterministic: once the replica is disconnected, - # old WAL can be recycled aggressively. - primary.append_conf( - wal_keep_size=0, - min_wal_size="32MB", - max_wal_size="64MB", - wal_sender_timeout="2s", - checkpoint_timeout="1h", - ) - primary.start() - - replica = lab.create_replica("p1", "r1", node_ip(lab, 11)) - replica.start() - replica.catchup() - - primary.execute("create table wal_gap(id int primary key)") - primary.execute("insert into wal_gap values (1)") - replica.catchup() - assert replica.execute("select array_agg(id order by id) from wal_gap") == [([1],)] - - # Capture the point at which both sides are known to be caught up. - disconnect_lsn = primary.execute( - "select pg_current_wal_lsn()::text" - )[0][0] - required_wal = primary.execute( - "select pg_walfile_name('{}'::pg_lsn)".format(disconnect_lsn) - )[0][0] - - assert primary.execute( - "select exists (select 1 from pg_ls_waldir() where name = '{}')".format( - required_wal - ) - ) == [(True,)] - - # 100% loss on replica egress drops ACKs/status replies. With the short - # wal_sender_timeout the primary closes the old streaming connection; all - # subsequent reconnect attempts from the replica are also dropped. - replica.netem(loss_pct=100) - - wait_until( - lambda: primary.execute(""" - select count(*) - from pg_stat_replication - where application_name = 'r1' - """)[0][0] == 0, - timeout=10, - desc="walsender to disappear after netem loss", - ) - - # No WAL is generated between the last catchup and the walsender going - # away. Now move sufficiently far ahead and checkpoint until the segment - # containing the replica's required position is really gone. We test the - # filesystem-visible WAL set rather than assuming that N switches are enough - # on every PostgreSQL version/configuration. - removed = False - switches = 0 - - for _ in range(8): - primary.execute( - "select pg_switch_wal() from generate_series(1, 4)" - ) - switches += 4 - primary.execute("checkpoint") - - exists = primary.execute( - "select exists (select 1 from pg_ls_waldir() where name = '{}')".format( - required_wal - ) - )[0][0] - - if not exists: - removed = True - break - - assert removed, ( - "required WAL {} was still present after {} forced WAL switches" - .format(required_wal, switches) - ) - - # Put an unmistakable change after the WAL gap. - primary.execute("insert into wal_gap values (2)") - target_lsn = primary.execute( - "select pg_current_wal_lsn()::text" - )[0][0] - - print( - "WAL gap created: disconnect_lsn={}, removed_wal={}, " - "switches={}, target_lsn={}".format( - disconnect_lsn, required_wal, switches, target_lsn - ) - ) - - # The network is healthy again, but the missing WAL is no longer available - # from the primary. Do NOT call Testgres catchup() here: its current - # implementation waits indefinitely (max_attempts=0) by design. - replica.clear_netem() - - # Give walreceiver several reconnect attempts. The replica itself is - # reachable from the controller again, so we can observe that replay does - # not cross the gap. - deadline = time.monotonic() + 8.0 - observed_counts = [] - - while time.monotonic() < deadline: - observed_counts.append( - replica.execute("select count(*) from wal_gap")[0][0] - ) - - caught_up = replica.execute( - "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn".format( - target_lsn - ) - )[0][0] - - assert not caught_up, ( - "replica unexpectedly crossed a WAL segment that was already " - "recycled on the primary" - ) - time.sleep(0.5) - - assert set(observed_counts) == {1}, observed_counts - assert primary.execute("select array_agg(id order by id) from wal_gap") == [([1, 2],)] - assert replica.execute("select array_agg(id order by id) from wal_gap") == [([1],)] - assert replica.execute("select pg_is_in_recovery()") == [(True,)] - - print( - "Replica stayed behind after network recovery as expected; " - "a new base backup or WAL archive is required." - ) From 5a8430bccd6082d687fa1935a1a852402e84c160 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 19 Oct 2026 15:04:45 +0300 Subject: [PATCH 11/11] Revert "Add PgLab Linux fault-injection support" This reverts commit d5fb01da836706ee3c23897edd70e8d083b05cbc. --- README.md | 28 -- src/pglab.py | 1111 -------------------------------------------------- 2 files changed, 1139 deletions(-) delete mode 100644 src/pglab.py diff --git a/README.md b/README.md index 99457a08..d615d367 100644 --- a/README.md +++ b/README.md @@ -120,34 +120,6 @@ with testgres.get_new_node('master') as master: print(replica.execute('postgres', 'select 1')) ``` -### Linux fault injection with PgLab - -`testgres.pglab` builds Linux-isolated PostgreSQL nodes on top of the normal -Testgres lifecycle. Each PgLab node gets its own network namespace and cgroup, -while Testgres still owns `initdb`, `pg_ctl`, `pg_basebackup`, replication and -SQL access. - -This is also a concrete use case for the target execution-context arguments of -`PostgresNode.replicate()`: a replica must be created with its own -`OsOperations`, host and port, and `pg_basebackup` must run in that target -context. - -```python -from testgres.pglab import PgLab - -with PgLab(pg_bin='/path/to/postgres/bin', os_user='postgres') as lab: - primary = lab.create_primary('p1', '10.203.0.10').init().start() - replica = lab.create_replica('p1', 'r1', '10.203.0.11').start() - replica.catchup() - - lab.partition('p1', 'r1') - lab.heal_partition('p1', 'r1') -``` - -PgLab is Linux-only and currently requires root privileges because it creates -network namespaces, veth devices and cgroups. Its privileged integration suite -is run separately in CI. - ### Benchmarks Use `pgbench` through `testgres` to run quick benchmarks: diff --git a/src/pglab.py b/src/pglab.py deleted file mode 100644 index 8ea3c798..00000000 --- a/src/pglab.py +++ /dev/null @@ -1,1111 +0,0 @@ -"""Linux fault-injection lab built on top of Testgres. - -PgLab keeps PostgreSQL lifecycle, backup and replication in Testgres while -placing individual nodes into separate Linux network namespaces and cgroups. -It is Linux-only and currently requires root privileges. -""" - -from __future__ import annotations - -import hashlib -import ipaddress -import os -import pwd -import signal -import shutil -import stat -import subprocess -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -from .api import get_new_node -from testgres.operations.local_ops import LocalOperations - - -class PgLabError(RuntimeError): - pass - - -def _run(cmd, check=True, capture=False): - kwargs = {"check": check, "text": True} - if capture: - kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return subprocess.run(list(cmd), **kwargs) - - -def _which(name): - path = shutil.which(name) - if not path: - raise PgLabError("required command not found: {}".format(name)) - return path - - -def _ifname(prefix, name): - # Linux IFNAMSIZ is 16 including the terminating NUL. - digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:6] - return (prefix + digest)[:15] - - -@dataclass -class IOLimits: - read_bps: Optional[int] = None - write_bps: Optional[int] = None - read_iops: Optional[int] = None - write_iops: Optional[int] = None - - def as_io_max(self, major, minor): - parts = [] - if self.read_bps is not None: - parts.append("rbps={}".format(self.read_bps)) - if self.write_bps is not None: - parts.append("wbps={}".format(self.write_bps)) - if self.read_iops is not None: - parts.append("riops={}".format(self.read_iops)) - if self.write_iops is not None: - parts.append("wiops={}".format(self.write_iops)) - if not parts: - return None - return "{}:{} {}".format(major, minor, " ".join(parts)) - - -class CgroupIOBackend: - version = None - - def setup_root(self): - raise NotImplementedError - - def create_node(self, name): - raise NotImplementedError - - def apply_limits(self, cgroup, device, limits): - raise NotImplementedError - - def membership_file(self, cgroup): - raise NotImplementedError - - def kill_node(self, cgroup): - pass - - def remove_node(self, cgroup): - cgroup.rmdir() - - def cleanup_root(self): - try: - self.root.rmdir() - except OSError: - pass - - -class CgroupV2IO(CgroupIOBackend): - version = 2 - - def __init__(self, root=Path("/sys/fs/cgroup/testgres-pglab")): - self.mount = Path("/sys/fs/cgroup") - self.root = Path(root) - - def setup_root(self): - controllers = (self.mount / "cgroup.controllers").read_text().split() - if "io" not in controllers: - raise PgLabError("cgroup v2 io controller is unavailable") - subtree = self.mount / "cgroup.subtree_control" - enabled = subtree.read_text().split() - if "io" not in enabled: - try: - subtree.write_text("+io\n") - except OSError as exc: - raise PgLabError("cannot enable cgroup v2 io controller: {}".format(exc)) - self.root.mkdir(exist_ok=True) - child_subtree = self.root / "cgroup.subtree_control" - available = (self.root / "cgroup.controllers").read_text().split() - if "io" in available and "io" not in child_subtree.read_text().split(): - child_subtree.write_text("+io\n") - - def create_node(self, name): - path = self.root / name - path.mkdir(exist_ok=True) - return path - - def apply_limits(self, cgroup, device, limits): - st = os.stat(device) - line = limits.as_io_max(os.major(st.st_rdev), os.minor(st.st_rdev)) - if line: - (cgroup / "io.max").write_text(line + "\n") - - def membership_file(self, cgroup): - return cgroup / "cgroup.procs" - - def kill_node(self, cgroup): - try: - (cgroup / "cgroup.kill").write_text("1\n") - except OSError: - pass - - -class CgroupV1Blkio(CgroupIOBackend): - version = 1 - - def __init__(self, mount=Path("/sys/fs/cgroup/blkio")): - self.mount = Path(mount) - self.root = self.mount / "testgres-pglab" - - def setup_root(self): - if not self.mount.is_dir(): - raise PgLabError("cgroup v1 blkio controller is unavailable") - self.root.mkdir(exist_ok=True) - - def create_node(self, name): - path = self.root / name - path.mkdir(exist_ok=True) - return path - - def _write_limit(self, cgroup, filename, major, minor, value): - if value is not None: - (cgroup / filename).write_text("{}:{} {}\n".format(major, minor, value)) - - def apply_limits(self, cgroup, device, limits): - st = os.stat(device) - major, minor = os.major(st.st_rdev), os.minor(st.st_rdev) - self._write_limit(cgroup, "blkio.throttle.read_bps_device", major, minor, limits.read_bps) - self._write_limit(cgroup, "blkio.throttle.write_bps_device", major, minor, limits.write_bps) - self._write_limit(cgroup, "blkio.throttle.read_iops_device", major, minor, limits.read_iops) - self._write_limit(cgroup, "blkio.throttle.write_iops_device", major, minor, limits.write_iops) - - def membership_file(self, cgroup): - return cgroup / "tasks" - - -def detect_cgroup_io_backend(): - # Pure/unified v2 has cgroup.controllers at the top-level mount. - if Path("/sys/fs/cgroup/cgroup.controllers").is_file(): - return CgroupV2IO() - # Ubuntu 20.04 commonly uses hybrid mode: cgroup2 is mounted under - # /sys/fs/cgroup/unified, while blkio remains on the v1 hierarchy. - if Path("/sys/fs/cgroup/blkio").is_dir(): - return CgroupV1Blkio() - raise PgLabError("no supported cgroup I/O controller found (need v2 io or v1 blkio)") - - -class NamespaceOperations(LocalOperations): - """Local Testgres operations whose *commands* execute in a netns. - - Filesystem operations intentionally remain ordinary local operations: a Linux - network namespace shares the host filesystem. PostgreSQL utilities launched - by Testgres (initdb, pg_ctl, psql, pg_basebackup, pg_isready, ...) are wrapped - in:: - - ip netns exec ... runuser -u -- - - The wrapper enters the node cgroup before dropping privileges, so postgres - and all of its descendants inherit the cgroup. - """ - - def __init__(self, namespace, os_user, db_user, cgroup=None, cgroup_membership_file=None, pg_bin=None): - super().__init__() - self.namespace = namespace - self.os_user = os_user - self.db_user = db_user - self.pg_bin = str(pg_bin) if pg_bin is not None else None - self.cgroup = Path(cgroup) if cgroup is not None else None - self.cgroup_membership_file = ( - Path(cgroup_membership_file) if cgroup_membership_file is not None else None - ) - - # Testgres uses os_ops.username as the default database username in a - # number of places. Keep it distinct from the Unix account we use in - # runuser. - self._username = db_user - - def environ(self, name): - # target_os_ops may be asked to resolve PostgreSQL binaries while a - # replica is being cloned. Expose the PgLab-specific bin directory to - # Testgres without requiring the caller to mutate the process-wide - # PG_BIN environment variable. - if name == "PG_BIN" and self.pg_bin is not None: - return self.pg_bin - return super().environ(name) - - def mkdtemp(self, prefix=None): - assert prefix is None or type(prefix) is str - - base = "/tmp/testgres-pglab-tmp" - os.makedirs(base, mode=0o755, exist_ok=True) - - path = tempfile.mkdtemp( - prefix=prefix, - dir=base, - ) - - pw = pwd.getpwnam(self.os_user) - os.chown(path, pw.pw_uid, pw.pw_gid) - os.chmod(path, 0o700) - - return path - - def mkstemp(self, prefix=None): - assert prefix is None or type(prefix) is str - - base = "/tmp/testgres-pglab-tmp" - os.makedirs(base, mode=0o755, exist_ok=True) - - fd, path = tempfile.mkstemp( - prefix=prefix, - dir=base, - ) - os.close(fd) - - pw = pwd.getpwnam(self.os_user) - os.chown(path, pw.pw_uid, pw.pw_gid) - os.chmod(path, 0o600) - - return path - - def create_clone(self): - return self.__class__( - namespace=self.namespace, - os_user=self.os_user, - db_user=self.db_user, - cgroup=str(self.cgroup) if self.cgroup is not None else None, - cgroup_membership_file=( - str(self.cgroup_membership_file) - if self.cgroup_membership_file is not None else None - ), - pg_bin=self.pg_bin, - ) - - def _command_needs_server_cgroup(self, payload): - """Return True only for commands that may start the postgres server. - - Client utilities such as psql, pg_basebackup and pg_isready should not - be charged to the server's I/O cgroup. pg_ctl inherits the cgroup and - the postmaster it starts keeps that membership. - """ - if not payload: - return False - exe = os.path.basename(str(payload[0])) - if exe == "postgres": - return True - if exe == "pg_ctl": - # Testgres normally invokes pg_ctl with an explicit action. Only - # start/restart can create a postmaster that needs to inherit the - # node cgroup. - args = [str(x) for x in payload[1:]] - return "start" in args or "restart" in args - return False - - def _with_pg_path(self, argv): - if self.pg_bin is None: - return list(argv) - path = self.pg_bin + os.pathsep + os.environ.get("PATH", "") - return ["env", "PATH=" + path] + list(argv) - - def _wrap_command(self, cmd, shell): - if shell: - if isinstance(cmd, str): - command = cmd - else: - command = self.join_command_arguments(list(cmd)) - payload = ["sh", "-c", command] - else: - if isinstance(cmd, str): - payload = ["sh", "-c", cmd] - else: - payload = list(cmd) - - prefix = ["ip", "netns", "exec", self.namespace] - run_as_user = ["runuser", "-u", self.os_user, "--"] + self._with_pg_path(payload) - - # Most Testgres utilities are clients. They belong to the controller, - # not to the PostgreSQL server cgroup. - if self.cgroup is None or not self._command_needs_server_cgroup(payload): - return prefix + run_as_user, False - - membership = self.cgroup_membership_file - if membership is None: - raise PgLabError("cgroup membership file was not configured") - if not membership.exists(): - raise PgLabError( - "node cgroup disappeared before postgres start: {}".format(membership) - ) - - # Join the cgroup in the host namespace first, then enter the network - # namespace. Some systems do not expose the host cgroup filesystem from - # an ip-netns helper mount namespace, while cgroup membership itself is - # inherited perfectly well across ip netns exec/runuser/pg_ctl. - script = ( - 'echo $$ > "$1"; ' - 'ns="$2"; user="$3"; shift 3; ' - 'exec ip netns exec "$ns" runuser -u "$user" -- "$@"' - ) - wrapped = [ - "sh", "-c", script, "pglab-cgroup", str(membership), - self.namespace, self.os_user, - ] + self._with_pg_path(payload) - return wrapped, False - - def exec_command(self, cmd, *args, **kwargs): - shell = kwargs.pop("shell", False) - wrapped, wrapped_shell = self._wrap_command(cmd, shell) - return super().exec_command(wrapped, *args, shell=wrapped_shell, **kwargs) - - def popen(self, cmd, *args, **kwargs): - shell = kwargs.pop("shell", False) - wrapped, wrapped_shell = self._wrap_command(cmd, shell) - return super().popen(wrapped, *args, shell=wrapped_shell, **kwargs) - - -class LabNode: - """A Linux-isolated node backed by a real testgres.PostgresNode.""" - - def __init__(self, lab, name, ip, namespace, veth_host, cgroup, - pg_node, os_ops, role): - self.lab = lab - self.name = name - self.ip = ip - self.namespace = namespace - self.veth_host = veth_host - self.veth_ns = "eth0" - self.cgroup = cgroup - self.pg = pg_node - self.os_ops = os_ops - self.role = role - - def __getattr__(self, name): - # Keep the normal Testgres API pleasant to use: - # node.execute(), node.safe_psql(), node.append_conf(), ... - return getattr(self.pg, name) - - def init(self, initdb_params=None, **kwargs): - params = list(initdb_params or []) - if "-U" not in params and "--username" not in params: - params += ["-U", self.lab.db_superuser] - self.lab._ensure_node_ownership(self) - self.pg.init(initdb_params=params, **kwargs) - - # Network namespaces isolate TCP/IP but share the host filesystem. - # Without this, nodes using the same PostgreSQL port collide on - # /tmp/.s.PGSQL. even though their TCP sockets are isolated. - self.pg.append_conf(unix_socket_directories="") - - self.lab._ensure_node_ownership(self) - return self - - def start(self, *args, **kwargs): - # Testgres may lazily create logs_dir during start(). Force creation - # while the controller is root, then hand the whole node tree to the - # configured OS user before pg_ctl is run through runuser. - _ = self.pg.logs_dir - self.lab._ensure_node_ownership(self) - self.pg.start(*args, **kwargs) - return self - - def stop(self, *args, **kwargs): - self.pg.stop(*args, **kwargs) - return self - - def network_down(self): - self.lab.network_down(self.name) - return self - - def network_up(self): - self.lab.network_up(self.name) - return self - - def netem(self, **kwargs): - self.lab.netem(self.name, **kwargs) - return self - - def clear_netem(self): - self.lab.clear_netem(self.name) - return self - - def freeze(self): - self.lab.freeze(self.name) - return self - - def unfreeze(self): - self.lab.unfreeze(self.name) - return self - - -class PgLab: - """Testgres + Linux network namespaces + cgroup I/O fault injection. - - PostgreSQL lifecycle is delegated to Testgres. PgLab owns only Linux - isolation (netns/veth/bridge/cgroup) and fault injection. - """ - - def __init__( - self, - pg_bin=None, - base_dir="/tmp/testgres-pglab", - subnet="10.203.0.0/24", - bridge_name="pglab0", - bridge_ip=None, - os_user="postgres", - db_superuser=None, - keep_data=False, - ): - self.pg_bin = str(pg_bin) if pg_bin is not None else None - self.base_dir = Path(base_dir) - self.subnet = ipaddress.ip_network(subnet, strict=True) - self.bridge_name = bridge_name - self.os_user = os_user - self.db_superuser = db_superuser or os_user - self.keep_data = keep_data - self.nodes = {} # type: Dict[str, LabNode] - self._created_bridge = False - self._closed = False - self._firewall_backend = None - self._firewall_source_added = False - self._iptables_rule_added = False - - - hosts = self.subnet.hosts() - default_bridge_ip = str(next(hosts)) - self.bridge_ip = bridge_ip or default_bridge_ip - if ipaddress.ip_address(self.bridge_ip) not in self.subnet: - raise PgLabError("bridge IP is outside lab subnet: {}".format(self.bridge_ip)) - - self._preflight() - self.base_dir.mkdir(parents=True, exist_ok=True) - self._setup_cgroups() - self._ensure_bridge() - self._setup_firewall() - - def _preflight(self): - if os.name != "posix" or not Path("/proc/self/ns/net").exists(): - raise PgLabError("pglab requires Linux network namespaces") - if os.geteuid() != 0: - raise PgLabError("pglab must currently be run as root") - _which("ip") - _which("runuser") - _which("findmnt") - try: - pwd.getpwnam(self.os_user) - except KeyError as exc: - raise PgLabError("OS user does not exist: {}".format(self.os_user)) from exc - # cgroup mode is detected separately. Ubuntu 20.04 commonly uses - # hybrid mode with blkio on cgroup v1. - if self.pg_bin is not None: - required = ("initdb", "pg_ctl", "postgres", "psql", "pg_basebackup") - for exe in required: - p = Path(self.pg_bin) / exe - if not p.is_file() or not os.access(str(p), os.X_OK): - raise PgLabError("PostgreSQL executable missing/not executable: {}".format(p)) - - def _setup_cgroups(self): - self.cgroup_backend = detect_cgroup_io_backend() - self.cgroup_backend.setup_root() - self.cgroup_root = self.cgroup_backend.root - - def _ensure_bridge(self): - exists = subprocess.run( - ["ip", "link", "show", "dev", self.bridge_name], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ).returncode == 0 - if not exists: - _run(["ip", "link", "add", self.bridge_name, "type", "bridge"]) - self._created_bridge = True - - # The host gets an address in the lab subnet. This is important because - # testgres' Python DB driver connects from the controller namespace to - # node.host; unlike psql, execute()/connect() do not go through os_ops. - addr = "{}/{}".format(self.bridge_ip, self.subnet.prefixlen) - _run(["ip", "addr", "replace", addr, "dev", self.bridge_name]) - _run(["ip", "link", "set", self.bridge_name, "up"]) - - def _setup_firewall(self): - """Allow traffic between PgLab nodes without changing persistent rules. - - firewalld and iptables-nft can be independent forwarding layers. When - firewalld is active we add the lab subnet to the runtime trusted zone, - and independently add a temporary iptables FORWARD allow rule. - """ - source = str(self.subnet) - firewall_cmd = shutil.which("firewall-cmd") - - if firewall_cmd is not None: - state = subprocess.run( - [firewall_cmd, "--state"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - if state.returncode == 0: - zone_proc = subprocess.run( - [firewall_cmd, "--get-zone-of-source={}".format(source)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - zone = zone_proc.stdout.strip() - if zone and zone != "trusted": - raise PgLabError( - "lab subnet {} already belongs to firewalld zone {!r}; " - "refusing to move an existing firewall configuration".format( - source, zone - ) - ) - if zone != "trusted": - proc = subprocess.run( - [ - firewall_cmd, - "--zone=trusted", - "--add-source={}".format(source), - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - ) - if proc.returncode != 0: - raise PgLabError( - "failed to add lab subnet {} to firewalld trusted zone: {}".format( - source, proc.stderr.strip() - ) - ) - self._firewall_source_added = True - self._firewall_backend = "firewalld" - - # firewalld and iptables-nft are not alternatives on many developer - # hosts. Docker/libvirt may leave an iptables FORWARD base chain with - # policy DROP at nft priority 0, while firewalld evaluates forwarding - # again in a later base chain (commonly priority +10). Both layers - # therefore need a temporary allow for the lab subnet. - iptables = shutil.which("iptables") - if iptables is None: - return - - rule = [ - "FORWARD", "-s", source, "-d", source, "-j", "ACCEPT", - ] - exists = subprocess.run( - [iptables, "-C"] + rule, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ).returncode == 0 - if not exists: - _run([iptables, "-I"] + rule) - self._iptables_rule_added = True - - if self._firewall_backend is None: - self._firewall_backend = "iptables" - - def _cleanup_firewall(self): - source = str(self.subnet) - - # The two firewall layers are independent and can both be active. - # Remove only rules that this PgLab instance actually added. - if self._firewall_source_added: - firewall_cmd = shutil.which("firewall-cmd") - if firewall_cmd is not None: - subprocess.run( - [ - firewall_cmd, - "--zone=trusted", - "--remove-source={}".format(source), - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - self._firewall_source_added = False - - if self._iptables_rule_added: - iptables = shutil.which("iptables") - if iptables is not None: - subprocess.run( - [ - iptables, "-D", "FORWARD", - "-s", source, "-d", source, - "-j", "ACCEPT", - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - self._iptables_rule_added = False - - @staticmethod - def _sysfs_block_path(device): - st = os.stat(device) - if not stat.S_ISBLK(st.st_mode): - raise PgLabError("not a block device: {}".format(device)) - return Path("/sys/dev/block/{}:{}".format(os.major(st.st_rdev), os.minor(st.st_rdev))) - - @staticmethod - def _device_from_sysfs(sysdev): - # sysdev may be a /sys/dev/block/M:m symlink or a path reached through - # a device-mapper/md 'slaves' directory. The kernel exposes DEVNAME - # in uevent for both disks and partitions. - uevent = Path(sysdev) / "uevent" - if not uevent.is_file(): - raise PgLabError("cannot resolve block device from sysfs: {}".format(sysdev)) - values = {} - for line in uevent.read_text().splitlines(): - if "=" in line: - k, v = line.split("=", 1) - values[k] = v - devname = values.get("DEVNAME") - if not devname: - raise PgLabError("DEVNAME is missing in {}".format(uevent)) - return "/dev/{}".format(devname) - - def _leaf_block_devices(self, device): - """Return physical queue devices suitable for blkio throttling. - - findmnt may return a partition or a stacked device such as LVM/dm-crypt. - cgroup-v1 blkio throttling can reject such a major:minor with ENODEV. - Walk sysfs 'slaves' recursively and, for partitions, use the parent disk. - """ - start = self._sysfs_block_path(device).resolve() - found = {} - - def visit(sysdev): - sysdev = Path(sysdev).resolve() - slaves = sysdev / "slaves" - if slaves.is_dir(): - children = list(slaves.iterdir()) - if children: - for child in children: - visit(child) - return - - # A partition has its own major:minor, but blkio v1 throttling is - # most portable when applied to the whole queue-bearing disk. - if (sysdev / "partition").is_file(): - parent = sysdev.parent - if (parent / "dev").is_file(): - sysdev = parent - - dev = self._device_from_sysfs(sysdev) - st = os.stat(dev) - key = (os.major(st.st_rdev), os.minor(st.st_rdev)) - found[key] = dev - - visit(start) - return [found[k] for k in sorted(found)] - - def _backing_block_devices(self, path): - res = _run(["findmnt", "-n", "-o", "SOURCE", "-T", str(path)], capture=True) - source = res.stdout.strip() - if not source.startswith("/dev/"): - raise PgLabError( - "cannot infer block device for {}: {!r}; pass io_device".format(path, source) - ) - devices = self._leaf_block_devices(source) - if not devices: - raise PgLabError("cannot resolve backing block device for {}".format(source)) - return devices - - def _apply_io_limits(self, cgroup, devices, limits): - if isinstance(devices, (str, Path)): - devices = [str(devices)] - for device in devices: - st = os.stat(device) - if not stat.S_ISBLK(st.st_mode): - raise PgLabError("not a block device: {}".format(device)) - self.cgroup_backend.apply_limits(cgroup, device, limits) - - def _ensure_node_ownership(self, node): - pw = pwd.getpwnam(self.os_user) - base = Path(node.pg.base_dir) - base.mkdir(parents=True, exist_ok=True) - for root, dirs, files in os.walk(str(base)): - os.chown(root, pw.pw_uid, pw.pw_gid) - for d in dirs: - os.chown(os.path.join(root, d), pw.pw_uid, pw.pw_gid) - for f in files: - try: - os.chown(os.path.join(root, f), pw.pw_uid, pw.pw_gid) - except FileNotFoundError: - pass - - def create_node(self, name, ip, role="standalone", io=None, io_device=None, port=5432): - if name in self.nodes: - raise PgLabError("node already exists: {}".format(name)) - if role not in ("primary", "replica", "standalone"): - raise PgLabError("invalid role: {}".format(role)) - - addr_obj = ipaddress.ip_address(ip) - if addr_obj not in self.subnet: - raise PgLabError("{} is outside lab subnet {}".format(ip, self.subnet)) - if ip == self.bridge_ip: - raise PgLabError("{} is reserved for the pglab bridge".format(ip)) - if any(n.ip == ip for n in self.nodes.values()): - raise PgLabError("IP already in use: {}".format(ip)) - - ns = "pglab-{}".format(name) - vh = _ifname("ph", name) - peer = _ifname("pn", name) - cgroup = self.cgroup_root / name - node_base = self.base_dir / name - - created_ns = False - created_veth = False - created_cgroup = False - try: - _run(["ip", "netns", "add", ns]) - created_ns = True - _run(["ip", "link", "add", vh, "type", "veth", "peer", "name", peer]) - created_veth = True - _run(["ip", "link", "set", peer, "netns", ns]) - _run(["ip", "link", "set", vh, "master", self.bridge_name]) - _run(["ip", "link", "set", vh, "up"]) - _run(["ip", "-n", ns, "link", "set", "lo", "up"]) - _run(["ip", "-n", ns, "link", "set", peer, "name", "eth0"]) - _run(["ip", "-n", ns, "addr", "add", - "{}/{}".format(ip, self.subnet.prefixlen), "dev", "eth0"]) - _run(["ip", "-n", ns, "link", "set", "eth0", "up"]) - - node_base.mkdir(parents=True, exist_ok=True) - pw = pwd.getpwnam(self.os_user) - os.chown(str(node_base), pw.pw_uid, pw.pw_gid) - - cgroup = self.cgroup_backend.create_node(name) - created_cgroup = True - if io is not None: - devices = [io_device] if io_device else self._backing_block_devices(node_base) - self._apply_io_limits(cgroup, devices, io) - - os_ops = NamespaceOperations( - namespace=ns, - os_user=self.os_user, - db_user=self.db_superuser, - cgroup=cgroup, - cgroup_membership_file=self.cgroup_backend.membership_file(cgroup), - pg_bin=self.pg_bin, - ) - - kwargs = { - "name": name, - "base_dir": str(node_base), - "port": int(port), - "host": ip, - "os_ops": os_ops, - } - if self.pg_bin is not None: - kwargs["bin_dir"] = self.pg_bin - pg_node = get_new_node(**kwargs) - - node = LabNode( - lab=self, - name=name, - ip=ip, - namespace=ns, - veth_host=vh, - cgroup=cgroup, - pg_node=pg_node, - os_ops=os_ops, - role=role, - ) - self.nodes[name] = node - return node - except Exception: - if created_veth: - subprocess.run(["ip", "link", "del", vh], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - if created_ns: - subprocess.run(["ip", "netns", "del", ns], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - if created_cgroup: - try: - cgroup.rmdir() - except OSError: - pass - if not self.keep_data: - shutil.rmtree(str(node_base), ignore_errors=True) - raise - - def create_primary(self, name, ip, **kwargs): - return self.create_node(name, ip, role="primary", **kwargs) - - def create_replica(self, primary_name, name, ip, **kwargs): - """Create a physical replica in its own network namespace. - - Current Testgres can run pg_basebackup directly in ``target_os_ops``. - PgLab creates the namespace/cgroup first, then asks Testgres to create - the replica through that execution context. The base backup therefore - originates from the replica namespace instead of being copied into it - afterwards. - """ - primary = self.nodes[primary_name] - replica_shell = self.create_node(name, ip, role="replica", **kwargs) - - target_ops = replica_shell.os_ops - target_port = replica_shell.pg.port - target_base = replica_shell.pg.base_dir - - # create_node() constructs a placeholder PostgresNode so PgLab can - # allocate the namespace, cgroup, address, port and storage location - # before replication. Reuse that base directory for the target-side - # base backup; this keeps replica data under PgLab.base_dir (important - # when PGLAB_TEST_ROOT selects a block-backed filesystem for I/O tests). - try: - spawned = primary.pg.replicate( - name=name, - target_os_ops=target_ops, - target_host=ip, - target_port=target_port, - username=self.db_superuser, - base_dir=target_base, - ) - except Exception: - self.remove_node(name) - raise - - replica_shell.pg = spawned - self._ensure_node_ownership(replica_shell) - return replica_shell - - def network_down(self, name): - node = self.nodes[name] - _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "down"]) - - def network_up(self, name): - node = self.nodes[name] - _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "up"]) - - @staticmethod - def _percent(value, name): - value = float(value) - if value < 0.0 or value > 100.0: - raise PgLabError("{} must be between 0 and 100".format(name)) - return value - - def netem( - self, - name, - delay_ms=None, - jitter_ms=None, - loss_pct=None, - duplicate_pct=None, - corrupt_pct=None, - reorder_pct=None, - rate_kbit=None, - ): - """Replace the node's root qdisc with a tc-netem fault profile. - - The profile is applied to traffic leaving the node's eth0 inside its - network namespace. Calling netem() again replaces the previous profile. - - Examples: - lab.netem("r1", delay_ms=100) - lab.netem("r1", delay_ms=100, jitter_ms=20, loss_pct=5) - lab.netem("r1", rate_kbit=1024) - lab.clear_netem("r1") - """ - node = self.nodes[name] - _which("tc") - - args = [ - "ip", "netns", "exec", node.namespace, - "tc", "qdisc", "replace", "dev", node.veth_ns, - "root", "netem", - ] - - have_fault = False - - if delay_ms is not None: - delay_ms = float(delay_ms) - if delay_ms < 0: - raise PgLabError("delay_ms must be >= 0") - args += ["delay", "{}ms".format(delay_ms)] - have_fault = True - - if jitter_ms is not None: - jitter_ms = float(jitter_ms) - if jitter_ms < 0: - raise PgLabError("jitter_ms must be >= 0") - args += ["{}ms".format(jitter_ms)] - elif jitter_ms is not None: - raise PgLabError("jitter_ms requires delay_ms") - - if loss_pct is not None: - loss_pct = self._percent(loss_pct, "loss_pct") - args += ["loss", "{}%".format(loss_pct)] - have_fault = True - - if duplicate_pct is not None: - duplicate_pct = self._percent(duplicate_pct, "duplicate_pct") - args += ["duplicate", "{}%".format(duplicate_pct)] - have_fault = True - - if corrupt_pct is not None: - corrupt_pct = self._percent(corrupt_pct, "corrupt_pct") - args += ["corrupt", "{}%".format(corrupt_pct)] - have_fault = True - - if reorder_pct is not None: - reorder_pct = self._percent(reorder_pct, "reorder_pct") - if delay_ms is None: - raise PgLabError("reorder_pct requires delay_ms") - args += ["reorder", "{}%".format(reorder_pct)] - have_fault = True - - if rate_kbit is not None: - rate_kbit = int(rate_kbit) - if rate_kbit <= 0: - raise PgLabError("rate_kbit must be > 0") - args += ["rate", "{}kbit".format(rate_kbit)] - have_fault = True - - if not have_fault: - raise PgLabError("netem requires at least one fault parameter") - - _run(args) - - def clear_netem(self, name): - """Remove the root qdisc fault profile from a node, if present.""" - node = self.nodes[name] - if shutil.which("tc") is None: - return - subprocess.run( - [ - "ip", "netns", "exec", node.namespace, - "tc", "qdisc", "del", "dev", node.veth_ns, "root", - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - - def _node_cgroup_pids(self, node): - membership = Path(node.os_ops.cgroup_membership_file) - try: - lines = membership.read_text().splitlines() - except FileNotFoundError: - return [] - pids = [] - for line in lines: - line = line.strip() - if line.isdigit(): - pids.append(int(line)) - return sorted(set(pids)) - - def freeze(self, name): - """SIGSTOP every task currently in the node's PostgreSQL cgroup. - - Unlike a network partition this leaves sockets and processes alive, but - the PostgreSQL node stops getting CPU time. This models a wedged VM, - scheduler stall, or process group that has stopped responding. - """ - node = self.nodes[name] - pids = self._node_cgroup_pids(node) - if not pids: - raise PgLabError("node cgroup has no processes to freeze: {}".format(name)) - - stopped = [] - try: - for pid in pids: - try: - os.kill(pid, signal.SIGSTOP) - stopped.append(pid) - except ProcessLookupError: - pass - except Exception: - # Do not leave a partially frozen node after a failed injection. - for pid in stopped: - try: - os.kill(pid, signal.SIGCONT) - except ProcessLookupError: - pass - raise - - def unfreeze(self, name): - """SIGCONT all tasks currently present in the node cgroup.""" - node = self.nodes[name] - for pid in self._node_cgroup_pids(node): - try: - os.kill(pid, signal.SIGCONT) - except ProcessLookupError: - pass - - def partition(self, left, right): - lnode = self.nodes[left] - rnode = self.nodes[right] - _run(["ip", "-n", lnode.namespace, "route", "replace", "blackhole", - "{}/32".format(rnode.ip)]) - _run(["ip", "-n", rnode.namespace, "route", "replace", "blackhole", - "{}/32".format(lnode.ip)]) - - def heal_partition(self, left, right): - lnode = self.nodes[left] - rnode = self.nodes[right] - subprocess.run(["ip", "-n", lnode.namespace, "route", "del", "blackhole", - "{}/32".format(rnode.ip)], check=False) - subprocess.run(["ip", "-n", rnode.namespace, "route", "del", "blackhole", - "{}/32".format(lnode.ip)], check=False) - - def set_io_limits(self, name, limits, io_device=None): - node = self.nodes[name] - devices = [io_device] if io_device else self._backing_block_devices(Path(node.pg.base_dir)) - self._apply_io_limits(node.cgroup, devices, limits) - - def remove_node(self, name): - node = self.nodes.get(name) - if node is None: - return - - # A frozen PostgreSQL cannot cooperate with pg_ctl stop/cleanup. - # Restore process scheduling and remove traffic shaping first. - try: - self.unfreeze(name) - except Exception: - pass - try: - self.clear_netem(name) - except Exception: - pass - - # PgLab removes the whole node directory itself below. Using - # PostgresNode.cleanup() here would perform another stop attempt first. - # That is noisy for two intentional PgLab states: an uninitialized node - # and a postmaster that a fault-injection test already killed. stop() - # itself always stops Testgres' log reader in its finally block, so call - # it directly, tolerate the expected failure, and release Testgres-owned - # resources before removing the isolation objects. - try: - node.pg.stop() - except Exception: - pass - try: - node.pg.release_resources() - except Exception: - pass - - subprocess.run(["ip", "netns", "del", node.namespace], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["ip", "link", "del", node.veth_host], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - self.cgroup_backend.kill_node(node.cgroup) - try: - self.cgroup_backend.remove_node(node.cgroup) - except OSError: - pass - if not self.keep_data: - shutil.rmtree(str(self.base_dir / name), ignore_errors=True) - del self.nodes[name] - - def close(self): - if self._closed: - return - for name in list(self.nodes.keys()): - self.remove_node(name) - self._cleanup_firewall() - if self._created_bridge: - subprocess.run(["ip", "link", "del", self.bridge_name], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - self.cgroup_backend.cleanup_root() - if not self.keep_data: - try: - self.base_dir.rmdir() - except OSError: - pass - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb): - self.close() - return False