From 4b7ea77f20edf989911a3ac67419134976084da2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 8 Oct 2021 18:07:57 +0100 Subject: [PATCH 1/6] Fail if an invalid X-option is provided in the command line --- Include/internal/pycore_initconfig.h | 3 +++ Lib/test/test_cmd_line.py | 13 +++++++++++-- Python/initconfig.c | 18 ++++++++++++++++++ Python/preconfig.c | 25 +++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 9014fcd41d8686..bb09424a5ac58c 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -82,6 +82,9 @@ PyAPI_FUNC(int) _Py_str_to_int( PyAPI_FUNC(const wchar_t*) _Py_get_xoption( const PyWideStringList *xoptions, const wchar_t *name); +PyAPI_FUNC(const wchar_t*) _Py_check_xoptions( + const PyWideStringList *xoptions, + const wchar_t **name); PyAPI_FUNC(const char*) _Py_GetEnv( int use_environment, const char *name); diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index d93e98f372532f..9e04e9fe9b919c 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -83,8 +83,17 @@ def get_xoptions(*args): opts = get_xoptions() self.assertEqual(opts, {}) - opts = get_xoptions('-Xa', '-Xb=c,d=e') - self.assertEqual(opts, {'a': True, 'b': 'c,d=e'}) + opts = get_xoptions('-Xno_debug_ranges') + self.assertEqual(opts, {'no_debug_ranges': True}) + + @unittest.skipIf(interpreter_requires_environment(), + 'Cannot run -E tests when PYTHON env vars are required.') + def test_unknown_xoptions(self): + rc, out, err = assert_python_failure('-X', 'blech') + self.assertIn(b'Unknown value for option -X', err) + msg = b'Fatal Python error: Unknown value for option -X' + self.assertEqual(err.splitlines().count(msg), 1) + self.assertEqual(b'', out) def test_showrefcount(self): def run_python(*args): diff --git a/Python/initconfig.c b/Python/initconfig.c index 2e3cde83b2534e..e68920759b01b2 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2121,6 +2121,19 @@ _PyConfig_InitImportConfig(PyConfig *config) return config_init_import(config, 1); } +const wchar_t* known_xoptions[] = { + L"faulthandler", + L"showrefcount", + L"tracemalloc", + L"importtime", + L"dev", + L"utf8", + L"pycache_prefix", + L"warn_default_encoding", + L"no_debug_ranges", + L"frozen_modules", + NULL, +}; static PyStatus config_read(PyConfig *config, int compute_path_config) @@ -2136,6 +2149,11 @@ config_read(PyConfig *config, int compute_path_config) } /* -X options */ + const wchar_t* option = _Py_check_xoptions(&config->xoptions, known_xoptions); + if (option != NULL) { + return PyStatus_Error("Unknown value for option -X"); + } + if (config_get_xoption(config, L"showrefcount")) { config->show_ref_count = 1; } diff --git a/Python/preconfig.c b/Python/preconfig.c index d59273159a6711..547bf97667829e 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -593,6 +593,31 @@ _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name) return NULL; } +const wchar_t* +_Py_check_xoptions(const PyWideStringList *xoptions, const wchar_t **names) +{ + for (Py_ssize_t i=0; i < xoptions->length; i++) { + const wchar_t *option = xoptions->items[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + int found = 0; + for (const wchar_t** name = names; *name != NULL; name++) { + if (wcsncmp(option, *name, len) == 0 && (*name)[len] == L'\0') { + found = 1; + } + } + if (found == 0) { + return option; + } + } + return NULL; +} static PyStatus preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) From 33bf9dc3a9a357aa602c65e6c9ca52d354fe6090 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 12 Oct 2021 14:01:40 +0100 Subject: [PATCH 2/6] fixup! Fail if an invalid X-option is provided in the command line --- Include/internal/pycore_initconfig.h | 3 --- Lib/test/test_audit.py | 4 +-- Lib/test/test_embed.py | 23 ++++++++-------- .../2021-10-12-14-41-39.bpo-45445._F5cMf.rst | 2 ++ Programs/_testembed.c | 18 ++++++------- Python/initconfig.c | 26 +++++++++++++++++++ Python/preconfig.c | 25 ------------------ 7 files changed, 50 insertions(+), 51 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index bb09424a5ac58c..9014fcd41d8686 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -82,9 +82,6 @@ PyAPI_FUNC(int) _Py_str_to_int( PyAPI_FUNC(const wchar_t*) _Py_get_xoption( const PyWideStringList *xoptions, const wchar_t *name); -PyAPI_FUNC(const wchar_t*) _Py_check_xoptions( - const PyWideStringList *xoptions, - const wchar_t **name); PyAPI_FUNC(const char*) _Py_GetEnv( int use_environment, const char *name); diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index c5ce26323b5f9e..d99b3b7ed7d36d 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -18,7 +18,7 @@ class AuditTest(unittest.TestCase): def do_test(self, *args): with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + [sys.executable, "-Xutf8", AUDIT_TESTS_PY, *args], encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -32,7 +32,7 @@ def do_test(self, *args): def run_python(self, *args): events = [] with subprocess.Popen( - [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + [sys.executable, "-Xutf8", AUDIT_TESTS_PY, *args], encoding="utf-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c748000f7005d1..d235ed0ba34c25 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -274,7 +274,7 @@ def test_pre_initialization_sys_options(self): "test_pre_initialization_sys_options", env=env) expected_output = ( "sys.warnoptions: ['once', 'module', 'default']\n" - "sys._xoptions: {'not_an_option': '1', 'also_not_an_option': '2'}\n" + "sys._xoptions: {'dev': '2', 'utf8': '1'}\n" "warnings.filters[:3]: ['default', 'module', 'once']\n" ) self.assertIn(expected_output, out) @@ -822,15 +822,14 @@ def test_init_from_config(self): 'argv': ['-c', 'arg2'], 'orig_argv': ['python3', '-W', 'cmdline_warnoption', - '-X', 'cmdline_xoption', + '-X', 'dev', '-c', 'pass', 'arg2'], 'parse_argv': 2, 'xoptions': [ - 'config_xoption1=3', - 'config_xoption2=', - 'config_xoption3', - 'cmdline_xoption', + 'dev=3', + 'utf8', + 'dev', ], 'warnoptions': [ 'cmdline_warnoption', @@ -1048,9 +1047,8 @@ def test_init_sys_add(self): config = { 'faulthandler': 1, 'xoptions': [ - 'config_xoption', - 'cmdline_xoption', - 'sysadd_xoption', + 'dev', + 'utf8', 'faulthandler', ], 'warnoptions': [ @@ -1060,9 +1058,12 @@ def test_init_sys_add(self): ], 'orig_argv': ['python3', '-W', 'ignore:::cmdline_warnoption', - '-X', 'cmdline_xoption'], + '-X', 'utf8'], } - self.check_all_configs("test_init_sys_add", config, api=API_PYTHON) + preconfig = {'utf8_mode': 1} + self.check_all_configs("test_init_sys_add", config, + expected_preconfig=preconfig, + api=API_PYTHON) def test_init_run_main(self): code = ('import _testinternalcapi, json; ' diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst new file mode 100644 index 00000000000000..3a57d8ba67143b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst @@ -0,0 +1,2 @@ +Python now fails to initialize if an invalid :option:`-X` option in the +command line. Patch by Pablo Galindo. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7628e1a17d9ff6..fa418e276114a2 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -211,7 +211,7 @@ static int test_pre_initialization_sys_options(void) * relying on the caller to keep the passed in strings alive. */ const wchar_t *static_warnoption = L"once"; - const wchar_t *static_xoption = L"also_not_an_option=2"; + const wchar_t *static_xoption = L"utf8=1"; size_t warnoption_len = wcslen(static_warnoption); size_t xoption_len = wcslen(static_xoption); wchar_t *dynamic_once_warnoption = \ @@ -230,7 +230,7 @@ static int test_pre_initialization_sys_options(void) PySys_AddWarnOption(L"module"); PySys_AddWarnOption(L"default"); _Py_EMBED_PREINIT_CHECK("Checking PySys_AddXOption\n"); - PySys_AddXOption(L"not_an_option=1"); + PySys_AddXOption(L"dev=2"); PySys_AddXOption(dynamic_xoption); /* Delete the dynamic options early */ @@ -548,7 +548,7 @@ static int test_init_from_config(void) L"-W", L"cmdline_warnoption", L"-X", - L"cmdline_xoption", + L"dev", L"-c", L"pass", L"arg2", @@ -556,10 +556,9 @@ static int test_init_from_config(void) config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); config.parse_argv = 1; - wchar_t* xoptions[3] = { - L"config_xoption1=3", - L"config_xoption2=", - L"config_xoption3", + wchar_t* xoptions[2] = { + L"dev=3", + L"utf8", }; config_set_wide_string_list(&config, &config.xoptions, Py_ARRAY_LENGTH(xoptions), xoptions); @@ -1375,7 +1374,6 @@ static int test_init_read_set(void) static int test_init_sys_add(void) { - PySys_AddXOption(L"sysadd_xoption"); PySys_AddXOption(L"faulthandler"); PySys_AddWarnOption(L"ignore:::sysadd_warnoption"); @@ -1387,14 +1385,14 @@ static int test_init_sys_add(void) L"-W", L"ignore:::cmdline_warnoption", L"-X", - L"cmdline_xoption", + L"utf8", }; config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); config.parse_argv = 1; PyStatus status; status = PyWideStringList_Append(&config.xoptions, - L"config_xoption"); + L"dev"); if (PyStatus_Exception(status)) { goto fail; } diff --git a/Python/initconfig.c b/Python/initconfig.c index e68920759b01b2..1d56b27a53382f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2135,6 +2135,32 @@ const wchar_t* known_xoptions[] = { NULL, }; +static const wchar_t* +_Py_check_xoptions(const PyWideStringList *xoptions, const wchar_t **names) +{ + for (Py_ssize_t i=0; i < xoptions->length; i++) { + const wchar_t *option = xoptions->items[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + int found = 0; + for (const wchar_t** name = names; *name != NULL; name++) { + if (wcsncmp(option, *name, len) == 0 && (*name)[len] == L'\0') { + found = 1; + } + } + if (found == 0) { + return option; + } + } + return NULL; +} + static PyStatus config_read(PyConfig *config, int compute_path_config) { diff --git a/Python/preconfig.c b/Python/preconfig.c index 547bf97667829e..d59273159a6711 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -593,31 +593,6 @@ _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name) return NULL; } -const wchar_t* -_Py_check_xoptions(const PyWideStringList *xoptions, const wchar_t **names) -{ - for (Py_ssize_t i=0; i < xoptions->length; i++) { - const wchar_t *option = xoptions->items[i]; - size_t len; - wchar_t *sep = wcschr(option, L'='); - if (sep != NULL) { - len = (sep - option); - } - else { - len = wcslen(option); - } - int found = 0; - for (const wchar_t** name = names; *name != NULL; name++) { - if (wcsncmp(option, *name, len) == 0 && (*name)[len] == L'\0') { - found = 1; - } - } - if (found == 0) { - return option; - } - } - return NULL; -} static PyStatus preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) From 20dd7561a9675b76b1ec5f1a68f28418825f827c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 12 Oct 2021 15:51:23 +0100 Subject: [PATCH 3/6] fixup! fixup! Fail if an invalid X-option is provided in the command line --- Python/initconfig.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index 1d56b27a53382f..6fba75b13b7ed2 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2121,6 +2121,10 @@ _PyConfig_InitImportConfig(PyConfig *config) return config_init_import(config, 1); } +// List of known xoptions to validate against the provided ones. Note that all +// options are listed, even if they are only available if a specific macro is +// set, like -X showrefcount which requires a debug build. In this case unknown +// options are silently ignored. const wchar_t* known_xoptions[] = { L"faulthandler", L"showrefcount", From ceedd70d84e6187fb6a8b7f2d6b211e0f1a43646 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 12 Oct 2021 23:19:41 +0100 Subject: [PATCH 4/6] Update Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst --- .../Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst index 3a57d8ba67143b..d497ae26bd5775 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst @@ -1,2 +1,2 @@ -Python now fails to initialize if an invalid :option:`-X` option in the +Python now fails to initialize if it finds an invalid :option:`-X` option in the command line. Patch by Pablo Galindo. From 19598b5e8b0cdb83117bd00ba34c0f8f413140c6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 13 Oct 2021 12:06:59 +0100 Subject: [PATCH 5/6] fixup! Update Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst --- Doc/library/sys.rst | 4 ++-- Lib/test/test_cmd_line.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ec12e02fb37d45..421af5b78cc60e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1725,13 +1725,13 @@ always available. .. code-block:: shell-session - $ ./python -Xa=b -Xc + $ ./python -Xpycache_prefix=some_path -Xdev Python 3.2a3+ (py3k, Oct 16 2010, 20:14:50) [GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import sys >>> sys._xoptions - {'a': 'b', 'c': True} + {'pycache_prefix': 'some_path', 'dev': True} .. impl-detail:: diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 9e04e9fe9b919c..1dc8c45885cbef 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -83,8 +83,8 @@ def get_xoptions(*args): opts = get_xoptions() self.assertEqual(opts, {}) - opts = get_xoptions('-Xno_debug_ranges') - self.assertEqual(opts, {'no_debug_ranges': True}) + opts = get_xoptions('-Xno_debug_ranges', '-Xdev=1234') + self.assertEqual(opts, {'no_debug_ranges': True, 'dev': '1234'}) @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -E tests when PYTHON env vars are required.') From dfd3e3ee9f78affb45a0f7d999fef1afdc243a77 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 13 Oct 2021 13:52:51 +0100 Subject: [PATCH 6/6] fixup! fixup! Update Misc/NEWS.d/next/Core and Builtins/2021-10-12-14-41-39.bpo-45445._F5cMf.rst --- test_foo.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 test_foo.py diff --git a/test_foo.py b/test_foo.py new file mode 100644 index 00000000000000..a27be0fdb47a69 --- /dev/null +++ b/test_foo.py @@ -0,0 +1,3 @@ +def foo(a=3, *, c, d=2): + pass +foo()