From 1ae7f8734c1c09ade809e4beb4fea5e8f153caae Mon Sep 17 00:00:00 2001 From: neonene Date: Thu, 9 Jun 2022 22:53:36 +0900 Subject: [PATCH 01/19] gh-91985: Ensure in-tree builds override platstdlib_dir in every path calculation --- Modules/getpath.py | 3 ++- Python/pathconfig.c | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 47f075caf5551b..45e9fcb10b94cf 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -461,7 +461,8 @@ def search_up(prefix, *landmarks, test=isfile): build_prefix = None -if not home_was_set and real_executable_dir and not py_setpath: +if ((not home_was_set and real_executable_dir and not py_setpath) + or config.get('_is_python_build')): # Detect a build marker and use it to infer prefix, exec_prefix, # stdlib_dir and the platstdlib_dir directories. try: diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 4271928571fa1f..7ba0e1b73440cc 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -36,10 +36,11 @@ typedef struct _PyPathConfig { wchar_t *program_name; /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ wchar_t *home; + int _is_python_build; } _PyPathConfig; # define _PyPathConfig_INIT \ - {.module_search_path = NULL} + {.module_search_path = NULL, ._is_python_build = 0} _PyPathConfig _Py_path_config = _PyPathConfig_INIT; @@ -72,6 +73,7 @@ _PyPathConfig_ClearGlobal(void) CLEAR(calculated_module_search_path); CLEAR(program_name); CLEAR(home); + _Py_path_config._is_python_build = 0; #undef CLEAR @@ -99,15 +101,24 @@ _PyPathConfig_ReadGlobal(PyConfig *config) } \ } while (0) +#define COPY_INT(ATTR) \ + do { \ + if ((_Py_path_config.ATTR > 0) && (config->ATTR <= 0)) { \ + config->ATTR = _Py_path_config.ATTR; \ + } \ + } while (0) + COPY(prefix); COPY(exec_prefix); COPY(stdlib_dir); COPY(program_name); COPY(home); COPY2(executable, program_full_path); + COPY_INT(_is_python_build); // module_search_path must be initialised - not read #undef COPY #undef COPY2 +#undef COPY_INT done: return status; @@ -137,14 +148,23 @@ _PyPathConfig_UpdateGlobal(const PyConfig *config) } \ } while (0) +#define COPY_INT(ATTR) \ + do { \ + if (config->ATTR > 0) { \ + _Py_path_config.ATTR = config->ATTR; \ + } \ + } while (0) + COPY(prefix); COPY(exec_prefix); COPY(stdlib_dir); COPY(program_name); COPY(home); COPY2(program_full_path, executable); + COPY_INT(_is_python_build); #undef COPY #undef COPY2 +#undef COPY_INT PyMem_RawFree(_Py_path_config.module_search_path); _Py_path_config.module_search_path = NULL; From 0be59f2ab5cefc81e50aee4104d3bb9ccdd0aad4 Mon Sep 17 00:00:00 2001 From: neonene Date: Sat, 11 Jun 2022 22:10:34 +0900 Subject: [PATCH 02/19] test, etc. --- Lib/test/test_getpath.py | 37 +++++++++++++++++++++++++++++++++++++ Modules/getpath.py | 2 +- Python/pathconfig.c | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 5208374e20013c..7ff2cb8cfba00f 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -239,6 +239,43 @@ def test_buildtree_pythonhome_win32(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_run_in_tree_build_win32(self): + "Test _is_python_build fields with edge cases.(gh-91985)" + def read_pathconfig(config, attr): + if _Py_path_config[attr] >= 0 and config[attr] <= 0: + config[attr] = _Py_path_config[attr] + + def update_pathconfig(config, attr): + if config[attr] > 0: + _Py_path_config[attr] = config[attr] + + edges = { + ( 0, -1) : 0, + (-1, 0) : -1, + (-1, -1) : -1, + ( 1, 0) : 1, + } + for key in edges: + _Py_path_config = dict( + _is_python_build=key[0], + ) + ns = MockNTNamespace( + argv0=r"C:\CPython\PCbuild\amd64\python.exe", + ) + ns['config'].update( + _is_python_build=key[1], + ) + # _PyConfig_InitPathConfig emulation + read_pathconfig(ns['config'], '_is_python_build') + expected = dict( + _is_python_build=ns['config']['_is_python_build'], + build_prefix=None, # no build-landmark + ) + actual = getpath(ns, expected) + self.assertEqual(actual, expected) + update_pathconfig(actual, '_is_python_build') + self.assertEqual(_Py_path_config['_is_python_build'], edges[key]) + def test_normal_posix(self): "Test a 'standard' install layout on *nix" ns = MockPosixNamespace( diff --git a/Modules/getpath.py b/Modules/getpath.py index 45e9fcb10b94cf..dceeed7702c0be 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -462,7 +462,7 @@ def search_up(prefix, *landmarks, test=isfile): build_prefix = None if ((not home_was_set and real_executable_dir and not py_setpath) - or config.get('_is_python_build')): + or config.get('_is_python_build', 0) > 0): # Detect a build marker and use it to infer prefix, exec_prefix, # stdlib_dir and the platstdlib_dir directories. try: diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 7ba0e1b73440cc..196a4b7f20ce2b 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -103,7 +103,7 @@ _PyPathConfig_ReadGlobal(PyConfig *config) #define COPY_INT(ATTR) \ do { \ - if ((_Py_path_config.ATTR > 0) && (config->ATTR <= 0)) { \ + if ((_Py_path_config.ATTR >= 0) && (config->ATTR <= 0)) { \ config->ATTR = _Py_path_config.ATTR; \ } \ } while (0) From 19fb85d22aa4950ead7022fcd65dd209cdb3e3dd Mon Sep 17 00:00:00 2001 From: neonene Date: Sat, 11 Jun 2022 22:33:24 +0900 Subject: [PATCH 03/19] less redundant test --- Lib/test/test_getpath.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 7ff2cb8cfba00f..c7ebfe423beead 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -249,21 +249,15 @@ def update_pathconfig(config, attr): if config[attr] > 0: _Py_path_config[attr] = config[attr] - edges = { - ( 0, -1) : 0, - (-1, 0) : -1, - (-1, -1) : -1, - ( 1, 0) : 1, - } - for key in edges: + for edges in [(0,-1), (-1, 0), (-1,-1), (1, 0)]: _Py_path_config = dict( - _is_python_build=key[0], + _is_python_build=edges[0], ) ns = MockNTNamespace( argv0=r"C:\CPython\PCbuild\amd64\python.exe", ) ns['config'].update( - _is_python_build=key[1], + _is_python_build=edges[1], ) # _PyConfig_InitPathConfig emulation read_pathconfig(ns['config'], '_is_python_build') @@ -274,7 +268,7 @@ def update_pathconfig(config, attr): actual = getpath(ns, expected) self.assertEqual(actual, expected) update_pathconfig(actual, '_is_python_build') - self.assertEqual(_Py_path_config['_is_python_build'], edges[key]) + self.assertEqual(_Py_path_config['_is_python_build'], edges[0]) def test_normal_posix(self): "Test a 'standard' install layout on *nix" From f336113b257d682943a301bd711946042b34a39b Mon Sep 17 00:00:00 2001 From: neonene Date: Sun, 12 Jun 2022 01:00:22 +0900 Subject: [PATCH 04/19] Fix test (place a landmark) --- Lib/test/test_getpath.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index c7ebfe423beead..0c0e431864b5dd 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -249,21 +249,23 @@ def update_pathconfig(config, attr): if config[attr] > 0: _Py_path_config[attr] = config[attr] - for edges in [(0,-1), (-1, 0), (-1,-1), (1, 0)]: + for edges in [(0,-1), (-1, 0), (-1,-1)]: _Py_path_config = dict( _is_python_build=edges[0], ) ns = MockNTNamespace( argv0=r"C:\CPython\PCbuild\amd64\python.exe", ) + ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) ns['config'].update( + home=r"C:\CPython", # turn on 'home_was_set' _is_python_build=edges[1], ) # _PyConfig_InitPathConfig emulation read_pathconfig(ns['config'], '_is_python_build') expected = dict( _is_python_build=ns['config']['_is_python_build'], - build_prefix=None, # no build-landmark + build_prefix=None, # landmarks must not be checked ) actual = getpath(ns, expected) self.assertEqual(actual, expected) From 0e75c9cd99192414b398e4a1780cd1cff05c472a Mon Sep 17 00:00:00 2001 From: neonene Date: Sun, 12 Jun 2022 13:52:05 +0900 Subject: [PATCH 05/19] assert() --- Lib/test/test_getpath.py | 2 +- Python/pathconfig.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 0c0e431864b5dd..5cdfb44ab6e39e 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -265,7 +265,7 @@ def update_pathconfig(config, attr): read_pathconfig(ns['config'], '_is_python_build') expected = dict( _is_python_build=ns['config']['_is_python_build'], - build_prefix=None, # landmarks must not be checked + build_prefix=None, # landmarks must not be checked for ) actual = getpath(ns, expected) self.assertEqual(actual, expected) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 196a4b7f20ce2b..69b7e10a3b0288 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -103,6 +103,7 @@ _PyPathConfig_ReadGlobal(PyConfig *config) #define COPY_INT(ATTR) \ do { \ + assert(_Py_path_config.ATTR >= 0); \ if ((_Py_path_config.ATTR >= 0) && (config->ATTR <= 0)) { \ config->ATTR = _Py_path_config.ATTR; \ } \ From 0df156bf470bca9d58f161957ce6ec87cfba2ba3 Mon Sep 17 00:00:00 2001 From: neonene Date: Sun, 12 Jun 2022 16:12:10 +0900 Subject: [PATCH 06/19] add case --- Lib/test/test_getpath.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 5cdfb44ab6e39e..347159c6da776b 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -249,7 +249,7 @@ def update_pathconfig(config, attr): if config[attr] > 0: _Py_path_config[attr] = config[attr] - for edges in [(0,-1), (-1, 0), (-1,-1)]: + for edges in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5, -10]]: _Py_path_config = dict( _is_python_build=edges[0], ) @@ -263,10 +263,17 @@ def update_pathconfig(config, attr): ) # _PyConfig_InitPathConfig emulation read_pathconfig(ns['config'], '_is_python_build') - expected = dict( - _is_python_build=ns['config']['_is_python_build'], - build_prefix=None, # landmarks must not be checked for - ) + if isinstance(edges, tuple): + expected = dict( + _is_python_build=ns['config']['_is_python_build'], + build_prefix=None, + ) + else: + edges = (1, 1) + expected = dict( + _is_python_build=edges[1], + build_prefix=r"C:\CPython", + ) actual = getpath(ns, expected) self.assertEqual(actual, expected) update_pathconfig(actual, '_is_python_build') From b085759c1c6b632a7192d4dfed546029f3ebcc2d Mon Sep 17 00:00:00 2001 From: neonene Date: Sun, 12 Jun 2022 17:08:59 +0900 Subject: [PATCH 07/19] add platstdlib_dir --- Lib/test/test_getpath.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 347159c6da776b..f2a3576597f246 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -249,7 +249,7 @@ def update_pathconfig(config, attr): if config[attr] > 0: _Py_path_config[attr] = config[attr] - for edges in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5, -10]]: + for edges in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5,-10]]: _Py_path_config = dict( _is_python_build=edges[0], ) @@ -258,7 +258,7 @@ def update_pathconfig(config, attr): ) ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) ns['config'].update( - home=r"C:\CPython", # turn on 'home_was_set' + home=r"C:\CPython", # turn on 'home_was_set' _is_python_build=edges[1], ) # _PyConfig_InitPathConfig emulation @@ -267,12 +267,14 @@ def update_pathconfig(config, attr): expected = dict( _is_python_build=ns['config']['_is_python_build'], build_prefix=None, + platstdlib_dir=r"C:\CPython\DLLs", ) else: edges = (1, 1) expected = dict( _is_python_build=edges[1], build_prefix=r"C:\CPython", + platstdlib_dir=r"C:\CPython\PCbuild\amd64", ) actual = getpath(ns, expected) self.assertEqual(actual, expected) From bc861bfe83b047f91b37afa4fe98ad3c34c97dd3 Mon Sep 17 00:00:00 2001 From: neonene Date: Mon, 13 Jun 2022 15:55:04 +0900 Subject: [PATCH 08/19] reword --- Lib/test/test_getpath.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f2a3576597f246..8e41a298a90b32 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -240,7 +240,7 @@ def test_buildtree_pythonhome_win32(self): self.assertEqual(expected, actual) def test_run_in_tree_build_win32(self): - "Test _is_python_build fields with edge cases.(gh-91985)" + "Test _is_python_build fields with invalid cases.(gh-91985)" def read_pathconfig(config, attr): if _Py_path_config[attr] >= 0 and config[attr] <= 0: config[attr] = _Py_path_config[attr] @@ -249,9 +249,9 @@ def update_pathconfig(config, attr): if config[attr] > 0: _Py_path_config[attr] = config[attr] - for edges in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5,-10]]: + for flags in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5,-10]]: _Py_path_config = dict( - _is_python_build=edges[0], + _is_python_build=flags[0], ) ns = MockNTNamespace( argv0=r"C:\CPython\PCbuild\amd64\python.exe", @@ -259,27 +259,27 @@ def update_pathconfig(config, attr): ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) ns['config'].update( home=r"C:\CPython", # turn on 'home_was_set' - _is_python_build=edges[1], + _is_python_build=flags[1], ) # _PyConfig_InitPathConfig emulation read_pathconfig(ns['config'], '_is_python_build') - if isinstance(edges, tuple): + if isinstance(flags, tuple): expected = dict( _is_python_build=ns['config']['_is_python_build'], build_prefix=None, platstdlib_dir=r"C:\CPython\DLLs", ) else: - edges = (1, 1) + flags = (1, 1) expected = dict( - _is_python_build=edges[1], + _is_python_build=flags[1], build_prefix=r"C:\CPython", platstdlib_dir=r"C:\CPython\PCbuild\amd64", ) actual = getpath(ns, expected) self.assertEqual(actual, expected) update_pathconfig(actual, '_is_python_build') - self.assertEqual(_Py_path_config['_is_python_build'], edges[0]) + self.assertEqual(_Py_path_config['_is_python_build'], flags[0]) def test_normal_posix(self): "Test a 'standard' install layout on *nix" From 5cb5e2bce2c908df1461f7ea78fca2b69a3a4b0a Mon Sep 17 00:00:00 2001 From: neonene Date: Mon, 13 Jun 2022 16:01:02 +0900 Subject: [PATCH 09/19] remove test --- Lib/test/test_getpath.py | 42 ---------------------------------------- 1 file changed, 42 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 8e41a298a90b32..5208374e20013c 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -239,48 +239,6 @@ def test_buildtree_pythonhome_win32(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) - def test_run_in_tree_build_win32(self): - "Test _is_python_build fields with invalid cases.(gh-91985)" - def read_pathconfig(config, attr): - if _Py_path_config[attr] >= 0 and config[attr] <= 0: - config[attr] = _Py_path_config[attr] - - def update_pathconfig(config, attr): - if config[attr] > 0: - _Py_path_config[attr] = config[attr] - - for flags in [(0,-1), (-1, 0), (-1,-1), (-10, 0), [-10, 5], [5,-10]]: - _Py_path_config = dict( - _is_python_build=flags[0], - ) - ns = MockNTNamespace( - argv0=r"C:\CPython\PCbuild\amd64\python.exe", - ) - ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) - ns['config'].update( - home=r"C:\CPython", # turn on 'home_was_set' - _is_python_build=flags[1], - ) - # _PyConfig_InitPathConfig emulation - read_pathconfig(ns['config'], '_is_python_build') - if isinstance(flags, tuple): - expected = dict( - _is_python_build=ns['config']['_is_python_build'], - build_prefix=None, - platstdlib_dir=r"C:\CPython\DLLs", - ) - else: - flags = (1, 1) - expected = dict( - _is_python_build=flags[1], - build_prefix=r"C:\CPython", - platstdlib_dir=r"C:\CPython\PCbuild\amd64", - ) - actual = getpath(ns, expected) - self.assertEqual(actual, expected) - update_pathconfig(actual, '_is_python_build') - self.assertEqual(_Py_path_config['_is_python_build'], flags[0]) - def test_normal_posix(self): "Test a 'standard' install layout on *nix" ns = MockPosixNamespace( From 3051b5548118d3ce8c4079c4f9155fa31fe08a3f Mon Sep 17 00:00:00 2001 From: neonene Date: Mon, 13 Jun 2022 21:55:31 +0900 Subject: [PATCH 10/19] new test case in test_embed --- Lib/test/test_embed.py | 42 ++++++++++++++++++++++++++++++++++++++++++ Programs/_testembed.c | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index f1ca6da147376c..ea27d97dbe7b6d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1303,6 +1303,48 @@ def test_init_setpythonhome(self): self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) + @unittest.skipUnless(MS_WINDOWS, 'Windows only') + def test_init_is_python_build_win32(self): + # Test _Py_path_config._is_python_build field (gh-91985) + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + + for path in paths: + if not os.path.isdir(path): + continue + if os.path.exists(os.path.join(path, 'os.py')): + home = os.path.dirname(path) + break + else: + self.fail(f"Unable to find home in {paths!r}") + + prefix = exec_prefix = home + stdlib = os.path.join(home, "Lib") + expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + config = { + 'home': home, + 'module_search_paths': expected_paths, + 'prefix': prefix, + 'base_prefix': prefix, + 'exec_prefix': exec_prefix, + 'base_exec_prefix': exec_prefix, + 'pythonpath_env': paths_str, + 'stdlib_dir': stdlib, + } + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + + env['INVALID_ISPYTHONBUILD'] = '1' + config['_is_python_build'] = 0 + self.check_all_configs("test_init_is_python_build", config, + api=API_COMPAT, env=env) + + env['INVALID_ISPYTHONBUILD'] = '0' + config['_is_python_build'] = 1 + config['module_search_paths'][-1] = os.path.dirname(self.test_exe) + self.check_all_configs("test_init_is_python_build", config, + api=API_COMPAT, env=env) + def copy_paths_by_env(self, config): all_configs = self._get_expected_config() paths = all_configs['config']['module_search_paths'] diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 9d3d0cbddf0e53..3503e73eb01124 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1550,6 +1550,46 @@ static int test_init_setpythonhome(void) } +static int test_init_is_python_build(void) +{ + char *env = getenv("TESTHOME"); + if (!env) { + error("missing TESTHOME env var"); + return 1; + } + wchar_t *home = Py_DecodeLocale(env, NULL); + if (home == NULL) { + error("failed to decode TESTHOME"); + return 1; + } + + PyConfig config; + _PyConfig_InitCompatConfig(&config); + config_set_program_name(&config); + // Ensure 'home_was_set' is turned on in getpath.py + config_set_string(&config, &config.home, home); + PyMem_RawFree(home); + putenv("TESTHOME="); + + config._is_python_build = INT_MAX; + env = getenv("INVALID_ISPYTHONBUILD"); + if (env) { + if (strcmp(env, "1") == 0) { + config._is_python_build++; + } + putenv("INVALID_ISPYTHONBUILD="); + } + init_from_config_clear(&config); + Py_Finalize(); + // Second initialization + config._is_python_build = -1; + init_from_config_clear(&config); + dump_config(); // home and _is_python_build are from _Py_path_config + Py_Finalize(); + return 0; +} + + static int test_init_warnoptions(void) { putenv("PYTHONWARNINGS=ignore:::env1,ignore:::env2"); @@ -1965,6 +2005,7 @@ static struct TestCase TestCases[] = { {"test_init_setpath", test_init_setpath}, {"test_init_setpath_config", test_init_setpath_config}, {"test_init_setpythonhome", test_init_setpythonhome}, + {"test_init_is_python_build", test_init_is_python_build}, {"test_init_warnoptions", test_init_warnoptions}, {"test_init_set_config", test_init_set_config}, {"test_run_main", test_run_main}, From 4f08be90b16f5d4c255a053bdf8b6f2a2ac35d45 Mon Sep 17 00:00:00 2001 From: neonene Date: Mon, 13 Jun 2022 22:35:43 +0900 Subject: [PATCH 11/19] reword --- Lib/test/test_embed.py | 4 ++-- Programs/_testembed.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index ea27d97dbe7b6d..647bcd3b1d29a3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1334,12 +1334,12 @@ def test_init_is_python_build_win32(self): } env = {'TESTHOME': home, 'PYTHONPATH': paths_str} - env['INVALID_ISPYTHONBUILD'] = '1' + env['NEGATIVE_ISPYTHONBUILD'] = '1' config['_is_python_build'] = 0 self.check_all_configs("test_init_is_python_build", config, api=API_COMPAT, env=env) - env['INVALID_ISPYTHONBUILD'] = '0' + env['NEGATIVE_ISPYTHONBUILD'] = '0' config['_is_python_build'] = 1 config['module_search_paths'][-1] = os.path.dirname(self.test_exe) self.check_all_configs("test_init_is_python_build", config, diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 3503e73eb01124..0171b752371320 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1572,12 +1572,12 @@ static int test_init_is_python_build(void) putenv("TESTHOME="); config._is_python_build = INT_MAX; - env = getenv("INVALID_ISPYTHONBUILD"); + env = getenv("NEGATIVE_ISPYTHONBUILD"); if (env) { if (strcmp(env, "1") == 0) { config._is_python_build++; } - putenv("INVALID_ISPYTHONBUILD="); + putenv("NEGATIVE_ISPYTHONBUILD="); } init_from_config_clear(&config); Py_Finalize(); From 6bb3323dcad252ec8d0f939fc56152d8bc9cc3e8 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 13 Jun 2022 23:03:56 +0900 Subject: [PATCH 12/19] rerun test --- Programs/_testembed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 0171b752371320..0e40605cd067ee 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1574,7 +1574,7 @@ static int test_init_is_python_build(void) config._is_python_build = INT_MAX; env = getenv("NEGATIVE_ISPYTHONBUILD"); if (env) { - if (strcmp(env, "1") == 0) { + if (strcmp(env, "0") != 0) { config._is_python_build++; } putenv("NEGATIVE_ISPYTHONBUILD="); From 19e14285c20aba9db9bedac40eb41c2c58b608e3 Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 02:17:33 +0900 Subject: [PATCH 13/19] apply suggested changes --- Lib/test/test_embed.py | 34 +++++++++++++++++++++++++++++----- Programs/_testembed.c | 15 +++++++-------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 647bcd3b1d29a3..b6187f47800024 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1303,8 +1303,7 @@ def test_init_setpythonhome(self): self.check_all_configs("test_init_setpythonhome", config, api=API_COMPAT, env=env) - @unittest.skipUnless(MS_WINDOWS, 'Windows only') - def test_init_is_python_build_win32(self): + def test_init_is_python_build_with_home(self): # Test _Py_path_config._is_python_build field (gh-91985) config = self._get_expected_config() paths = config['config']['module_search_paths'] @@ -1320,8 +1319,16 @@ def test_init_is_python_build_win32(self): self.fail(f"Unable to find home in {paths!r}") prefix = exec_prefix = home - stdlib = os.path.join(home, "Lib") - expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + if MS_WINDOWS: + stdlib = os.path.join(home, "Lib") + # Because we are specifying 'home', module search paths + # are fairly static + expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] + else: + version = f'{sys.version_info.major}.{sys.version_info.minor}' + stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + config = { 'home': home, 'module_search_paths': expected_paths, @@ -1341,7 +1348,24 @@ def test_init_is_python_build_win32(self): env['NEGATIVE_ISPYTHONBUILD'] = '0' config['_is_python_build'] = 1 - config['module_search_paths'][-1] = os.path.dirname(self.test_exe) + + exedir = os.path.dirname(sys.executable) + with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: + platstdlib = os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]) + platstdlib = os.path.normpath(platstdlib) + + if MS_WINDOWS: + expected_paths = [paths[0], stdlib, platstdlib] + else: + prefix = exec_prefix = sys.prefix + expected_paths = [self.module_search_paths(prefix=prefix)[0], + stdlib, platstdlib] + # stdlib calculation (/Lib) is not yet supported. + stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + + config.update(module_search_paths=expected_paths, + prefix=prefix, base_prefix=prefix, + exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) self.check_all_configs("test_init_is_python_build", config, api=API_COMPAT, env=env) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 0e40605cd067ee..d60f4b5a19ff91 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1566,25 +1566,24 @@ static int test_init_is_python_build(void) PyConfig config; _PyConfig_InitCompatConfig(&config); config_set_program_name(&config); - // Ensure 'home_was_set' is turned on in getpath.py + // gh-91985: in-tree builds fail to check for build directory landmarks + // under the effect of 'home' or PYTHONHOME environment variable. config_set_string(&config, &config.home, home); PyMem_RawFree(home); - putenv("TESTHOME="); + // Use an impossible value so we can detect whether it isn't updated + // during initialization. config._is_python_build = INT_MAX; env = getenv("NEGATIVE_ISPYTHONBUILD"); - if (env) { - if (strcmp(env, "0") != 0) { - config._is_python_build++; - } - putenv("NEGATIVE_ISPYTHONBUILD="); + if (env && strcmp(env, "0") != 0) { + config._is_python_build++; } init_from_config_clear(&config); Py_Finalize(); // Second initialization config._is_python_build = -1; init_from_config_clear(&config); - dump_config(); // home and _is_python_build are from _Py_path_config + dump_config(); // home and _is_python_build are cached in _Py_path_config Py_Finalize(); return 0; } From 19195906a67f1bb5e7292e1cba05eeb142338446 Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 03:00:26 +0900 Subject: [PATCH 14/19] move stdlib upper --- Lib/test/test_embed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index b6187f47800024..83a17873d72fdd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1358,10 +1358,10 @@ def test_init_is_python_build_with_home(self): expected_paths = [paths[0], stdlib, platstdlib] else: prefix = exec_prefix = sys.prefix - expected_paths = [self.module_search_paths(prefix=prefix)[0], - stdlib, platstdlib] # stdlib calculation (/Lib) is not yet supported. stdlib = os.path.join(home, sys.platlibdir, f'python{version}') + expected_paths = [self.module_search_paths(prefix=prefix)[0], + stdlib, platstdlib] config.update(module_search_paths=expected_paths, prefix=prefix, base_prefix=prefix, From 8b1591780bc6c10072381a14aaf4d26daedeb949 Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 16:33:07 +0900 Subject: [PATCH 15/19] comment, etc. --- Lib/test/test_embed.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 83a17873d72fdd..75b86853b2b022 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1304,7 +1304,7 @@ def test_init_setpythonhome(self): api=API_COMPAT, env=env) def test_init_is_python_build_with_home(self): - # Test _Py_path_config._is_python_build field (gh-91985) + # Test _Py_path_config._is_python_build configuration (gh-91985) config = self._get_expected_config() paths = config['config']['module_search_paths'] paths_str = os.path.pathsep.join(paths) @@ -1348,24 +1348,20 @@ def test_init_is_python_build_with_home(self): env['NEGATIVE_ISPYTHONBUILD'] = '0' config['_is_python_build'] = 1 - exedir = os.path.dirname(sys.executable) with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: - platstdlib = os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]) - platstdlib = os.path.normpath(platstdlib) - + platstdlib = os.path.normpath(os.path.join(exedir, + f'{f.read()}\n$'.splitlines()[0])) if MS_WINDOWS: - expected_paths = [paths[0], stdlib, platstdlib] + expected_paths[2] = platstdlib else: + # PREFIX (default) is set when running in build directory prefix = exec_prefix = sys.prefix - # stdlib calculation (/Lib) is not yet supported. - stdlib = os.path.join(home, sys.platlibdir, f'python{version}') - expected_paths = [self.module_search_paths(prefix=prefix)[0], - stdlib, platstdlib] - - config.update(module_search_paths=expected_paths, - prefix=prefix, base_prefix=prefix, - exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) + # stdlib calculation (/Lib) is not yet supported + expected_paths[0] = self.module_search_paths(prefix=prefix)[0] + expected_paths[2] = platstdlib + config.update(prefix=prefix, base_prefix=prefix, + exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) self.check_all_configs("test_init_is_python_build", config, api=API_COMPAT, env=env) From bf5bc288c992e46beea3722a9584f3fda8d2ddee Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 17:03:36 +0900 Subject: [PATCH 16/19] less lines --- Lib/test/test_embed.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 75b86853b2b022..650fb6bbf4274c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1350,16 +1350,13 @@ def test_init_is_python_build_with_home(self): config['_is_python_build'] = 1 exedir = os.path.dirname(sys.executable) with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f: - platstdlib = os.path.normpath(os.path.join(exedir, - f'{f.read()}\n$'.splitlines()[0])) - if MS_WINDOWS: - expected_paths[2] = platstdlib - else: + expected_paths[2] = os.path.normpath( + os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) + if not MS_WINDOWS: # PREFIX (default) is set when running in build directory prefix = exec_prefix = sys.prefix # stdlib calculation (/Lib) is not yet supported expected_paths[0] = self.module_search_paths(prefix=prefix)[0] - expected_paths[2] = platstdlib config.update(prefix=prefix, base_prefix=prefix, exec_prefix=exec_prefix, base_exec_prefix=exec_prefix) self.check_all_configs("test_init_is_python_build", config, From 39c851c37c9d1288ded2df6233611089cbc6b63d Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 18:24:17 +0900 Subject: [PATCH 17/19] putenv() --- Programs/_testembed.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d60f4b5a19ff91..826cbd04b081e1 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1570,6 +1570,7 @@ static int test_init_is_python_build(void) // under the effect of 'home' or PYTHONHOME environment variable. config_set_string(&config, &config.home, home); PyMem_RawFree(home); + putenv("TESTHOME="); // Use an impossible value so we can detect whether it isn't updated // during initialization. From 500cb811d08cad804ae914087ea6c28d89f2eaa2 Mon Sep 17 00:00:00 2001 From: neonene Date: Wed, 15 Jun 2022 22:25:04 +0900 Subject: [PATCH 18/19] move comment upper --- Programs/_testembed.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 826cbd04b081e1..542e46968ce564 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1552,6 +1552,8 @@ static int test_init_setpythonhome(void) static int test_init_is_python_build(void) { + // gh-91985: in-tree builds fail to check for build directory landmarks + // under the effect of 'home' or PYTHONHOME environment variable. char *env = getenv("TESTHOME"); if (!env) { error("missing TESTHOME env var"); @@ -1566,8 +1568,6 @@ static int test_init_is_python_build(void) PyConfig config; _PyConfig_InitCompatConfig(&config); config_set_program_name(&config); - // gh-91985: in-tree builds fail to check for build directory landmarks - // under the effect of 'home' or PYTHONHOME environment variable. config_set_string(&config, &config.home, home); PyMem_RawFree(home); putenv("TESTHOME="); From 7636f73d77ee16e52f7b3fbeaabc0d2e7dd832b5 Mon Sep 17 00:00:00 2001 From: neonene Date: Thu, 16 Jun 2022 23:53:34 +0900 Subject: [PATCH 19/19] a comment --- Lib/test/test_embed.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 650fb6bbf4274c..0229cf3c3127f8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1339,6 +1339,7 @@ def test_init_is_python_build_with_home(self): 'pythonpath_env': paths_str, 'stdlib_dir': stdlib, } + # The code above is taken from test_init_setpythonhome() env = {'TESTHOME': home, 'PYTHONPATH': paths_str} env['NEGATIVE_ISPYTHONBUILD'] = '1'