From 726edc7fd02daebd92d69539b4508f032f073f07 Mon Sep 17 00:00:00 2001 From: Stepan Neretin Date: Thu, 20 Feb 2025 16:24:20 +0700 Subject: [PATCH 1/5] add change data dir option --- testgres/node.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 859fe742..c127606e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -441,7 +441,14 @@ def logs_dir(self): @property def data_dir(self): # NOTE: we can't run initdb without user's args - return os.path.join(self.base_dir, DATA_DIR) + if hasattr(self, '_data_dir'): + return self._data_dir + else: + return os.path.join(self.base_dir, DATA_DIR) + + @data_dir.setter + def data_dir(self, value): + self._data_dir = value @property def utils_log_file(self): From 57c98b1ec6cbab7c64c6156d25db2dc0ad95189f Mon Sep 17 00:00:00 2001 From: Stepan Neretin Date: Thu, 20 Feb 2025 16:42:21 +0700 Subject: [PATCH 2/5] add daemonize run binary option --- .../pg_probackup2/pg_probackup2/app.py | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 57492814..1224d5ab 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -45,6 +45,7 @@ class ProbackupApp: def __init__(self, test_class: unittest.TestCase, pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): + self.process = None self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path @@ -61,7 +62,7 @@ def __init__(self, test_class: unittest.TestCase, self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, - skip_log_directory=False, expect_error=False, use_backup_dir=True): + skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): """ Run pg_probackup backup_dir: target directory for making backup @@ -118,11 +119,35 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() - self.test_class.output = subprocess.check_output( - cmdline, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8', errors='replace') + if daemonize: + + def stream_output(stream: subprocess.PIPE) -> None: + for line in iter(stream.readline, ''): + print(line) + stream.close() + + self.process = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env + ) + logging.info(f"Process started in background with PID: {self.process.pid}") + + if self.process.stdout and self.process.stderr: + stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,)) + stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,)) + + stdout_thread.start() + stderr_thread.start() + return self.process.pid + else: + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') end_time = time.time() self.execution_time = end_time - start_time From b7595ade357d330571cce354846103bda568a4ee Mon Sep 17 00:00:00 2001 From: Stepan Neretin Date: Fri, 28 Feb 2025 14:42:40 +0700 Subject: [PATCH 3/5] add ability assert logs --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1224d5ab..3003e43a 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -124,6 +124,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, def stream_output(stream: subprocess.PIPE) -> None: for line in iter(stream.readline, ''): print(line) + self.test_class.output += line stream.close() self.process = subprocess.Popen( From e63371a47e7e7de15c36734e684b23031e0b9838 Mon Sep 17 00:00:00 2001 From: Stepan Neretin Date: Tue, 4 Mar 2025 17:12:34 +0700 Subject: [PATCH 4/5] probackup plugin fixes --- .../pg_probackup2/pg_probackup2/app.py | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 3003e43a..d47cf51f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -61,6 +61,33 @@ def __init__(self, test_class: unittest.TestCase, self.test_class.output = None self.execution_time = None + def form_daemon_process(self, cmdline, env): + def stream_output(stream: subprocess.PIPE) -> None: + try: + for line in iter(stream.readline, ''): + print(line) + self.test_class.output += line + finally: + stream.close() + + self.process = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env + ) + logging.info(f"Process started in background with PID: {self.process.pid}") + + if self.process.stdout and self.process.stderr: + stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,), daemon=True) + stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + return self.process.pid + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): """ @@ -120,29 +147,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, start_time = time.time() if daemonize: - - def stream_output(stream: subprocess.PIPE) -> None: - for line in iter(stream.readline, ''): - print(line) - self.test_class.output += line - stream.close() - - self.process = subprocess.Popen( - cmdline, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env - ) - logging.info(f"Process started in background with PID: {self.process.pid}") - - if self.process.stdout and self.process.stderr: - stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,)) - stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,)) - - stdout_thread.start() - stderr_thread.start() - return self.process.pid + return self.form_daemon_process(cmdline, env) else: self.test_class.output = subprocess.check_output( cmdline, From a8f042cee4f09e4813881c7009743dc8bdd65fe4 Mon Sep 17 00:00:00 2001 From: Stepan Neretin Date: Wed, 19 Mar 2025 10:59:39 +0700 Subject: [PATCH 5/5] revert data dir setter changes because it not needed --- testgres/node.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index c127606e..859fe742 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -441,14 +441,7 @@ def logs_dir(self): @property def data_dir(self): # NOTE: we can't run initdb without user's args - if hasattr(self, '_data_dir'): - return self._data_dir - else: - return os.path.join(self.base_dir, DATA_DIR) - - @data_dir.setter - def data_dir(self, value): - self._data_dir = value + return os.path.join(self.base_dir, DATA_DIR) @property def utils_log_file(self):