From 2004fb6be321033a85e163c9e99d85aadb59f9d0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 15:26:26 +0200 Subject: [PATCH 1/5] gh-109276: libregrtest: WASM use filename for JSON On Emscripten and WASI platforms, libregrtest now uses a filename for the JSON file. Passing a file descriptor to a child process doesn't work on these platforms. --- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/run_workers.py | 35 ++++++++++++++++++----------- Lib/test/libregrtest/runtests.py | 19 +++++++++++++--- Lib/test/libregrtest/utils.py | 2 +- Lib/test/libregrtest/worker.py | 33 +++++++++++++++++---------- 5 files changed, 61 insertions(+), 30 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dd9a10764b075a..ba493ae1796fe0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -406,7 +406,7 @@ def create_run_tests(self, tests: TestTuple): python_cmd=self.python_cmd, randomize=self.randomize, random_seed=self.random_seed, - json_fd=None, + json_file=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index eaca0af17ea13a..07b90b00d2839a 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -17,7 +17,7 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests +from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, StrJSON, TestName, MS_WINDOWS, @@ -155,10 +155,11 @@ def mp_result_error( ) -> MultiprocessResult: return MultiprocessResult(test_result, stdout, err_msg) - def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int, + def _run_process(self, runtests: RunTests, output_fd: int, + json_file: JsonFileType, tmp_dir: StrPath | None = None) -> int: try: - popen = create_worker_process(runtests, output_fd, json_fd, + popen = create_worker_process(runtests, output_fd, json_file, tmp_dir) self._killed = False @@ -226,21 +227,29 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: match_tests = None err_msg = None + stdout_file = tempfile.TemporaryFile('w+', encoding=encoding) + if JSON_FILE_USE_FILENAME: + json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8') + else: + json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') + # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. - with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file, - tempfile.TemporaryFile('w+', encoding='utf8') as json_file): + with (stdout_file, json_tmpfile): stdout_fd = stdout_file.fileno() - json_fd = json_file.fileno() - if MS_WINDOWS: - json_fd = msvcrt.get_osfhandle(json_fd) + if JSON_FILE_USE_FILENAME: + json_file = json_tmpfile.name + else: + json_file = json_tmpfile.fileno() + if MS_WINDOWS: + json_file = msvcrt.get_osfhandle(json_file) kwargs = {} if match_tests: kwargs['match_tests'] = match_tests worker_runtests = self.runtests.copy( tests=tests, - json_fd=json_fd, + json_file=json_file, **kwargs) # gh-93353: Check for leaked temporary files in the parent process, @@ -254,13 +263,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: tmp_dir = os.path.abspath(tmp_dir) try: retcode = self._run_process(worker_runtests, - stdout_fd, json_fd, tmp_dir) + stdout_fd, json_file, tmp_dir) finally: tmp_files = os.listdir(tmp_dir) os_helper.rmtree(tmp_dir) else: retcode = self._run_process(worker_runtests, - stdout_fd, json_fd) + stdout_fd, json_file) tmp_files = () stdout_file.seek(0) @@ -275,8 +284,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: try: # deserialize run_tests_worker() output - json_file.seek(0) - worker_json: StrJSON = json_file.read() + json_tmpfile.seek(0) + worker_json: StrJSON = json_tmpfile.read() if worker_json: result = TestResult.from_json(worker_json) else: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 5c68df126e2a8f..7da86752ea7a37 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -2,10 +2,25 @@ import json from typing import Any +from test import support + from .utils import ( StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) +if support.is_emscripten or support.is_wasi: + # On Emscripten/WASI, it's a filename. Passing a file descriptor to a + # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the + # worker process. + JsonFileType = StrPath + JSON_FILE_USE_FILENAME = True +else: + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + JsonFileType = int + JSON_FILE_USE_FILENAME = False + + @dataclasses.dataclass(slots=True, frozen=True) class HuntRefleak: warmups: int @@ -38,9 +53,7 @@ class RunTests: python_cmd: tuple[str] | None randomize: bool random_seed: int | None - # On Unix, it's a file descriptor. - # On Windows, it's a handle. - json_fd: int | None + json_file: JsonFileType | None def copy(self, **override): state = dataclasses.asdict(self) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 03c27b9fe17053..69161b21bf3fbc 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -387,7 +387,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # testing (see the -j option). # Emscripten and WASI have stubbed getpid(), Emscripten has only # milisecond clock resolution. Use randint() instead. - if sys.platform in {"emscripten", "wasi"}: + if support.is_emscripten or support.is_wasi: nounce = random.randint(0, 1_000_000) else: nounce = os.getpid() diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 0963faa2e4d2a1..f2e60c89d80ed5 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper from .setup import setup_process, setup_test_dir -from .runtests import RunTests +from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME from .single import run_single_test from .utils import ( StrPath, StrJSON, FilterTuple, MS_WINDOWS, @@ -18,7 +18,7 @@ def create_worker_process(runtests: RunTests, - output_fd: int, json_fd: int, + output_fd: int, json_file: JsonFileType, tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -55,33 +55,42 @@ def create_worker_process(runtests: RunTests, close_fds=True, cwd=work_dir, ) - if not MS_WINDOWS: - kwargs['pass_fds'] = [json_fd] - else: + if JSON_FILE_USE_FILENAME: + # Nothing to do to pass the JSON filename to the worker process + pass + elif MS_WINDOWS: + # Pass the JSON handle to the worker process startupinfo = subprocess.STARTUPINFO() - startupinfo.lpAttributeList = {"handle_list": [json_fd]} + startupinfo.lpAttributeList = {"handle_list": [json_file]} kwargs['startupinfo'] = startupinfo + else: + # Pass the JSON file descriptor to the worker process + kwargs['pass_fds'] = [json_file] if USE_PROCESS_GROUP: kwargs['start_new_session'] = True if MS_WINDOWS: - os.set_handle_inheritable(json_fd, True) + os.set_handle_inheritable(json_file, True) try: return subprocess.Popen(cmd, **kwargs) finally: if MS_WINDOWS: - os.set_handle_inheritable(json_fd, False) + os.set_handle_inheritable(json_file, False) def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: FilterTuple | None = runtests.match_tests - json_fd: int = runtests.json_fd + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + # On Emscripten/WASI, it's a filename. + json_file: JsonFileType = runtests.json_file if MS_WINDOWS: import msvcrt - json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY) + # Create a file descriptor from the handle + json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY) setup_test_dir(runtests.test_dir) @@ -96,8 +105,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn: result = run_single_test(test_name, runtests) - with open(json_fd, 'w', encoding='utf-8') as json_file: - result.write_json_into(json_file) + with open(json_file, 'w', encoding='utf-8') as fp: + result.write_json_into(fp) sys.exit(0) From 9c052ba31c9cad8d85f33d3910e21320a8309be6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 15:55:54 +0200 Subject: [PATCH 2/5] DEBUG --- Lib/test/libregrtest/run_workers.py | 8 ++++++++ Lib/test/libregrtest/worker.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 07b90b00d2839a..a5a9935650c85b 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -227,11 +227,17 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: match_tests = None err_msg = None + print("main process is_emscripten:", support.is_emscripten) + print("main process is_wasi:", support.is_wasi) + print("main process JSON_FILE_USE_FILENAME:", JSON_FILE_USE_FILENAME) + stdout_file = tempfile.TemporaryFile('w+', encoding=encoding) if JSON_FILE_USE_FILENAME: json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8') + print("main process: create NamedTemporaryFile") else: json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') + print("main process: create TemporaryFile") # gh-94026: Write stdout+stderr to a tempfile as workaround for # non-blocking pipes on Emscripten with NodeJS. @@ -243,6 +249,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: json_file = json_tmpfile.fileno() if MS_WINDOWS: json_file = msvcrt.get_osfhandle(json_file) + print("main process json_type file:", type(json_file)) + print("main process json_type:", json_file) kwargs = {} if match_tests: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index f2e60c89d80ed5..b597b06cd31b4a 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -86,6 +86,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn: # On Windows, it's a handle. # On Emscripten/WASI, it's a filename. json_file: JsonFileType = runtests.json_file + print("worker: json_file type:", type(json_file)) + print("worker: json_file:", json_file) if MS_WINDOWS: import msvcrt From c4363703399e01768207a6e225dfa595bfe4e6d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 18:52:33 +0200 Subject: [PATCH 3/5] MORE DEBUG --- Lib/test/libregrtest/main.py | 5 +++++ Lib/test/libregrtest/run_workers.py | 4 ---- Lib/test/libregrtest/runtests.py | 5 +++++ Lib/test/libregrtest/worker.py | 4 ++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ba493ae1796fe0..3431b3187cff3f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -507,5 +507,10 @@ def main(self, tests: TestList | None = None): def main(tests=None, **kwargs): """Run the Python suite.""" + + print("main process start sys.platform:", sys.platform) + print("main process start is_emscripten:", support.is_emscripten) + print("main process start is_wasi:", support.is_wasi) + ns = _parse_args(sys.argv[1:], **kwargs) Regrtest(ns).main(tests=tests) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index a5a9935650c85b..bb96ca602f8f77 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -227,10 +227,6 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: match_tests = None err_msg = None - print("main process is_emscripten:", support.is_emscripten) - print("main process is_wasi:", support.is_wasi) - print("main process JSON_FILE_USE_FILENAME:", JSON_FILE_USE_FILENAME) - stdout_file = tempfile.TemporaryFile('w+', encoding=encoding) if JSON_FILE_USE_FILENAME: json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8') diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 7da86752ea7a37..112b185b664f06 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -20,6 +20,11 @@ JsonFileType = int JSON_FILE_USE_FILENAME = False +import os +print(os.getpid(), "JSON_FILE_USE_FILENAME:", JSON_FILE_USE_FILENAME) +print(os.getpid(), "JsonFileType:", JsonFileType) + + @dataclasses.dataclass(slots=True, frozen=True) class HuntRefleak: diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index b597b06cd31b4a..8af777213a3026 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -122,6 +122,10 @@ def main(): tmp_dir = get_temp_dir() work_dir = get_work_dir(tmp_dir, worker=True) + print("worker process sys.platform:", sys.platform) + print("worker process is_emscripten:", support.is_emscripten) + print("worker process is_wasi:", support.is_wasi) + with exit_timeout(): with os_helper.temp_cwd(work_dir, quiet=True): worker_process(worker_json) From 4c8d5c82fe3e812054d38a4689d146015d76aa12 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 19:13:24 +0200 Subject: [PATCH 4/5] I WANT MOOOOOAR DEBUG --- Lib/test/libregrtest/main.py | 1 + Lib/test/libregrtest/worker.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3431b3187cff3f..1d360958064c38 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -117,6 +117,7 @@ def __init__(self, ns: Namespace): self.python_cmd: tuple[str] = tuple(ns.python) else: self.python_cmd = None + print("main process python_cmd:", self.python_cmd) self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self.tmp_dir: StrPath | None = ns.tempdir diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 8af777213a3026..36853ff96e9b02 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -27,10 +27,12 @@ def create_worker_process(runtests: RunTests, executable = python_cmd else: executable = [sys.executable] + print("main process executable:", executable) cmd = [*executable, *support.args_from_interpreter_flags(), '-u', # Unbuffered stdout and stderr '-m', 'test.libregrtest.worker', worker_json] + print("main process worker cmd:", cmd) env = dict(os.environ) if tmp_dir is not None: From 4e2cfab30d2b660c43205168965df229707a2444 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 19:36:03 +0200 Subject: [PATCH 5/5] Fix the fix of the fixer fix --- Lib/test/libregrtest/run_workers.py | 21 +++++++++++++-------- Lib/test/libregrtest/runtests.py | 28 +++++++++++++++------------- Lib/test/libregrtest/utils.py | 10 +++++++--- Lib/test/libregrtest/worker.py | 16 +++++++++++----- 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index bb96ca602f8f77..5dfa9677b8b96c 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -17,10 +17,10 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME +from .runtests import RunTests, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, MS_WINDOWS, + StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX, format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP @@ -228,9 +228,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: err_msg = None stdout_file = tempfile.TemporaryFile('w+', encoding=encoding) - if JSON_FILE_USE_FILENAME: - json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8') - print("main process: create NamedTemporaryFile") + + json_file_use_filename = self.runtests.json_file_use_filename() + print("main process: json_file_use_filename?", json_file_use_filename) + if json_file_use_filename: + prefix = TMP_PREFIX + 'json_' + json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8', + prefix=prefix) + print("main process: create NamedTemporaryFile", json_tmpfile.name) else: json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') print("main process: create TemporaryFile") @@ -239,14 +244,14 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult: # non-blocking pipes on Emscripten with NodeJS. with (stdout_file, json_tmpfile): stdout_fd = stdout_file.fileno() - if JSON_FILE_USE_FILENAME: + if json_file_use_filename: json_file = json_tmpfile.name else: json_file = json_tmpfile.fileno() if MS_WINDOWS: json_file = msvcrt.get_osfhandle(json_file) - print("main process json_type file:", type(json_file)) - print("main process json_type:", json_file) + print("main process json_file type:", type(json_file)) + print("main process json_file:", json_file) kwargs = {} if match_tests: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 112b185b664f06..c61ef5390f0552 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -8,20 +8,10 @@ StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) -if support.is_emscripten or support.is_wasi: - # On Emscripten/WASI, it's a filename. Passing a file descriptor to a - # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the - # worker process. - JsonFileType = StrPath - JSON_FILE_USE_FILENAME = True -else: - # On Unix, it's a file descriptor. - # On Windows, it's a handle. - JsonFileType = int - JSON_FILE_USE_FILENAME = False +# See RunTests.json_file_use_filename() +JsonFileType = int | StrPath import os -print(os.getpid(), "JSON_FILE_USE_FILENAME:", JSON_FILE_USE_FILENAME) print(os.getpid(), "JsonFileType:", JsonFileType) @@ -58,7 +48,7 @@ class RunTests: python_cmd: tuple[str] | None randomize: bool random_seed: int | None - json_file: JsonFileType | None + json_file: JsonFileType | None def copy(self, **override): state = dataclasses.asdict(self) @@ -92,6 +82,18 @@ def as_json(self) -> StrJSON: def from_json(worker_json: StrJSON) -> 'RunTests': return json.loads(worker_json, object_hook=_decode_runtests) + def json_file_use_filename(self): + # On Unix, it's a file descriptor. + # On Windows, it's a handle. + # On Emscripten/WASI, it's a filename. Passing a file descriptor to a + # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the + # worker process. + return ( + self.python_cmd + or support.is_emscripten + or support.is_wasi + ) + class _EncodeRunTests(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 69161b21bf3fbc..ce7342aabfffbe 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -17,8 +17,12 @@ MS_WINDOWS = (sys.platform == 'win32') -WORK_DIR_PREFIX = 'test_python_' -WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_' + +# All temporary files and temporary directories created by libregrtest should +# use TMP_PREFIX so cleanup_temp_dir() can remove them all. +TMP_PREFIX = 'test_python_' +WORK_DIR_PREFIX = TMP_PREFIX +WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_' # bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). # Used to protect against threading._shutdown() hang. @@ -580,7 +584,7 @@ def display_header(): def cleanup_temp_dir(tmp_dir: StrPath): import glob - path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*') + path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') print("Cleanup %s directory" % tmp_dir) for name in glob.glob(path): if os.path.isdir(name): diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 36853ff96e9b02..f88e6dbc16f539 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper from .setup import setup_process, setup_test_dir -from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME +from .runtests import RunTests, JsonFileType from .single import run_single_test from .utils import ( StrPath, StrJSON, FilterTuple, MS_WINDOWS, @@ -57,17 +57,23 @@ def create_worker_process(runtests: RunTests, close_fds=True, cwd=work_dir, ) - if JSON_FILE_USE_FILENAME: - # Nothing to do to pass the JSON filename to the worker process + + # Pass json_file to the worker process + if isinstance(json_file, str): + # Filename: nothing to do to + print("create_worker_process() json_file: filename") pass elif MS_WINDOWS: - # Pass the JSON handle to the worker process + # Windows handle + print("create_worker_process() json_file: Windows handle") startupinfo = subprocess.STARTUPINFO() startupinfo.lpAttributeList = {"handle_list": [json_file]} kwargs['startupinfo'] = startupinfo else: - # Pass the JSON file descriptor to the worker process + # Unix file descriptor + print("create_worker_process() json_file: Unix fd") kwargs['pass_fds'] = [json_file] + if USE_PROCESS_GROUP: kwargs['start_new_session'] = True