bpo-2506: Add -X noopt command line option

Add -X noopt command line option to disable compiler optimizations. distutils ignores noopt: always enable compiler optimizations. * Add sys.flags.noopt flag. * Add 'noopt' keyword-only parameter to builtin compile(), importlib.util.cache_from_source() and py_compile.compile(). * importlib: SourceLoader gets an additional keyword-only '_noopt' parameter. It is used by py_compile.compile(). * importlib uses ``.noopt.pyc`` suffix for .pyc files if sys.flags.noopt is true. * Add PyConfig.optimize parameter. * Update subprocess._optim_args_from_interpreter_flags(). * Add support.requires_compiler_optimizations() * Update unit tests which rely on compiler optimizations for noopt: add @support.requires_compiler_optimizations. * Rename c_optimize to c_optimization_level in compiler.c Co-Authored-By: Yury Selivanov <yury@magic.io> Co-Authored-By: Victor Stinner <vstinner@python.org>
python · pablogsal · Aug 31, 2020 · Aug 31, 2020 · Aug 31, 2020 · Sep 2, 2020
commit 445778ff6643adac4406878be7814a57ea907362
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
@@ -580,6 +580,14 @@ PyConfig
       by the function calculating the :ref:`Path Configuration
       <init-path-config>`.
 
+   .. c:member:: int optimize
+
+      Should the compiler optimize bytecode? If equal to 0, disable the
+      compiler optimizations and set :c:member:`~PyConfig.optimization_level`
+      to 0. Set to 0 by :option:`-X noopt <-X>` command line option.
+
+      .. versionadded:: 3.10
+
    .. c:member:: int optimization_level
 
       Compilation optimization level:

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
@@ -242,7 +242,7 @@ are always available.  They are listed here in alphabetical order.
       Class methods can now wrap other :term:`descriptors <descriptor>` such as
       :func:`property`.
 
-.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1)
+.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1, noopt=None)
 
    Compile the *source* into a code or AST object.  Code objects can be executed
    by :func:`exec` or :func:`eval`.  *source* can either be a normal string, a
@@ -286,6 +286,9 @@ are always available.  They are listed here in alphabetical order.
    ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false)
    or ``2`` (docstrings are removed too).
 
+   If *noopt* is false, disable compiler optimizations and ignore *optimize*
+   argument. If it is ``None``, use ``sys.flags.noopt`` value.
+
    This function raises :exc:`SyntaxError` if the compiled source is invalid,
    and :exc:`ValueError` if the source contains null bytes.
 
@@ -319,10 +322,13 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
-   .. versionadded:: 3.8
+   .. versionchanged:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
       support for top-level ``await``, ``async for``, and ``async with``.
 
+   .. versionachanged:: 3.10
+      New *noopt* optional keyword-only parameter.
+
 
 .. class:: complex([real[, imag]])
 

diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
@@ -1410,7 +1410,7 @@ an :term:`importer`.
 
    .. versionadded:: 3.4
 
-.. function:: cache_from_source(path, debug_override=None, *, optimization=None)
+.. function:: cache_from_source(path, debug_override=None, *, optimization=None, noopt=None)
 
    Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated
    with the source *path*.  For example, if *path* is ``/foo/bar/baz.py`` the return
@@ -1429,6 +1429,11 @@ an :term:`importer`.
    ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation
    of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised.
 
+   The *noopt* parameter is used to specify if compiler optimization are
+   disabled. If it is true, *optimization* is ignored and ``.noopt`` suffix is
+   used (ex: ``baz.cpython-32.noopt.pyc``). If it is ``None``, use
+   ``sys.flags.noopt`` value.
+
    The *debug_override* parameter is deprecated and can be used to override
    the system's value for ``__debug__``. A ``True`` value is the equivalent of
    setting *optimization* to the empty string. A ``False`` value is the same as
@@ -1444,6 +1449,9 @@ an :term:`importer`.
    .. versionchanged:: 3.6
       Accepts a :term:`path-like object`.
 
+   .. versionchanged:: 3.10
+      Added *noopt* parameter.
+
 
 .. function:: source_from_cache(path)
 

diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst
@@ -60,6 +60,9 @@ byte-code cache files in the directory containing the source code.
    :func:`compile` function.  The default of ``-1`` selects the optimization
    level of the current interpreter.
 
+   If *noopt* is true, disable compiler optimizations and ignore *optimize*
+   argument. If it is ``None``, use ``sys.flags.noopt`` value.
+
    *invalidation_mode* should be a member of the :class:`PycInvalidationMode`
    enum and controls how the generated bytecode cache is invalidated at
    runtime.  The default is :attr:`PycInvalidationMode.CHECKED_HASH` if
@@ -92,6 +95,9 @@ byte-code cache files in the directory containing the source code.
    .. versionchanged:: 3.8
       The *quiet* parameter was added.
 
+   .. versionchanged:: 3.10
+      The *noopt* parameter was added.
+
 
 .. class:: PycInvalidationMode
 

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
@@ -439,6 +439,7 @@ always available.
    :const:`inspect`              :option:`-i`
    :const:`interactive`          :option:`-i`
    :const:`isolated`             :option:`-I`
+   :const:`noopt`                :option:`-X noopt <-X>`
    :const:`optimize`             :option:`-O` or :option:`-OO`
    :const:`dont_write_bytecode`  :option:`-B`
    :const:`no_user_site`         :option:`-s`
@@ -469,6 +470,9 @@ always available.
       Mode <devmode>` and the ``utf8_mode`` attribute for the new  :option:`-X`
       ``utf8`` flag.
 
+   .. versionchanged:: 3.10
+      Added ``noopt`` attribute for the new :option:`-X noopt <-X>` option.
+
 
 .. data:: float_info
 

diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
@@ -454,6 +454,7 @@ Miscellaneous options
    * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel
      tree rooted at the given directory instead of to the code tree. See also
      :envvar:`PYTHONPYCACHEPREFIX`.
+   * ``-X noopt`` disables the compiler optimizations.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -477,9 +478,9 @@ Miscellaneous options
       The ``-X pycache_prefix`` option. The ``-X dev`` option now logs
       ``close()`` exceptions in :class:`io.IOBase` destructor.
 
-   .. versionchanged:: 3.9
-      Using ``-X dev`` option, check *encoding* and *errors* arguments on
-      string encoding and decoding operations.
+   .. versionadded:: 3.10
+      The ``-X noopt`` option. Using ``-X dev`` option, check *encoding* and
+      *errors* arguments on string encoding and decoding operations.
 
       The ``-X showalloccount`` option has been removed.
 

diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst
@@ -93,6 +93,11 @@ Other Language Changes
   :meth:`~object.__index__` method).
   (Contributed by Serhiy Storchaka in :issue:`37999`.)
 
+* Added :option:`-X noopt <-X>` command line to disable compiler optimizations.
+  :mod:`importlib` uses ``.noopt.pyc`` suffix for ``.pyc`` filenames if
+  :data:`sys.flags.noopt` is true.
+  (Contributed by Yury Selivanov, Victor Stinner and Pablo Galindo in :issue:`2506`)
+
 
 New Modules
 ===========
@@ -109,6 +114,12 @@ base64
 Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the
 Base32 Encoding with Extended Hex Alphabet.
 
+builtins
+--------
+
+The :func:`compile` function gets a new optional keyword-only *noopt* parameter
+to disable compiler optimizations.
+
 curses
 ------
 
@@ -126,6 +137,20 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and
 :func:`~glob.iglob` which allow to specify the root directory for searching.
 (Contributed by Serhiy Storchaka in :issue:`38144`.)
 
+importlib
+---------
+
+Add a new *noopt* optional keyword-only parameter to
+:func:`importlib.util.cache_from_source` to disable compiler optimizations.
+(Contributed by Yury Selivanov, Victor Stinner and Pablo Galindo in :issue:`2506`)
+
+py_compile
+----------
+
+Add a new *noopt* optional keyword-only parameter to :func:`py_compile.compile`
+to disable compiler optimizations.
+(Contributed by Yury Selivanov, Victor Stinner and Pablo Galindo in :issue:`2506`)
+
 os
 --
 
@@ -145,6 +170,11 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line
 arguments passed to the Python executable.
 (Contributed by Victor Stinner in :issue:`23427`.)
 
+
+Add new :data:`sys.flags.noopt` flag for the new :option:`-X noopt <-X>` option
+(disable compiler optimizations).
+(Contributed by Yury Selivanov, Victor Stinner and Pablo Galindo in :issue:`2506`)
+
 xml
 ---
 

diff --git a/Include/compile.h b/Include/compile.h
@@ -106,4 +106,7 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeSta
 /* This doesn't need to match anything */
 #define Py_fstring_input 800
 
+/* Value of 'optimize' to deactivate all optimizations */
+#define _PyCompiler_disable_all_optimizations  -2
+
 #endif /* !Py_COMPILE_H */
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
@@ -417,6 +417,17 @@ typedef struct {
 
        _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */
     PyWideStringList orig_argv;
+
+    /* Should the compiler optimize bytecode?
+
+       If equal to 0, disable compiler optimizations and set optimization_level
+       to 0.
+
+       If equal to 1, enable compiler optimizations.
+
+       Set to 0 by -X noopt. */
+    int optimize;
+
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
@@ -62,6 +62,11 @@ PyAPI_FUNC(PyObject *) Py_CompileStringObject(
     PyObject *filename, int start,
     PyCompilerFlags *flags,
     int optimize);
+PyAPI_FUNC(PyObject *) _Py_CompileString(
+    const char *str,
+    PyObject *filename, int start,
+    PyCompilerFlags *flags,
+    int optimize);
 #endif
 PyAPI_FUNC(struct symtable *) Py_SymtableString(
     const char *str,

diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py
@@ -314,10 +314,10 @@ def get_outputs(self, include_bytecode=1):
             if include_bytecode:
                 if self.compile:
                     outputs.append(importlib.util.cache_from_source(
-                        filename, optimization=''))
+                        filename, optimization='', noopt=False))
                 if self.optimize > 0:
                     outputs.append(importlib.util.cache_from_source(
-                        filename, optimization=self.optimize))
+                        filename, optimization=self.optimize, noopt=False))
 
         outputs += [
             os.path.join(build_dir, filename)

diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py
@@ -166,10 +166,10 @@ def _bytecode_filenames(self, py_filenames):
                 continue
             if self.compile:
                 bytecode_files.append(importlib.util.cache_from_source(
-                    py_file, optimization=''))
+                    py_file, optimization='', noopt=False))
             if self.optimize > 0:
                 bytecode_files.append(importlib.util.cache_from_source(
-                    py_file, optimization=self.optimize))
+                    py_file, optimization=self.optimize, noopt=False))
 
         return bytecode_files
 

diff --git a/Lib/distutils/tests/test_install_lib.py b/Lib/distutils/tests/test_install_lib.py
@@ -44,9 +44,11 @@ def test_byte_compile(self):
         f = os.path.join(project_dir, 'foo.py')
         self.write_file(f, '# python file')
         cmd.byte_compile([f])
-        pyc_file = importlib.util.cache_from_source('foo.py', optimization='')
+        pyc_file = importlib.util.cache_from_source('foo.py', optimization='',
+                                                    noopt=False)
         pyc_opt_file = importlib.util.cache_from_source('foo.py',
-                                                    optimization=cmd.optimize)
+                                                    optimization=cmd.optimize,
+                                                    noopt=False)
         self.assertTrue(os.path.exists(pyc_file))
         self.assertTrue(os.path.exists(pyc_opt_file))
 

diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
@@ -444,10 +444,11 @@ def byte_compile (py_files,
             #   dfile - purported source filename (same as 'file' by default)
             if optimize >= 0:
                 opt = '' if optimize == 0 else optimize
-                cfile = importlib.util.cache_from_source(
-                    file, optimization=opt)
+                cfile = importlib.util.cache_from_source(file,
+                                                         optimization=opt,
+                                                         noopt=False)
             else:
-                cfile = importlib.util.cache_from_source(file)
+                cfile = importlib.util.cache_from_source(file, noopt=False)
             dfile = file
             if prefix:
                 if file[:len(prefix)] != prefix:
@@ -462,7 +463,7 @@ def byte_compile (py_files,
                 if force or newer(file, cfile):
                     log.info("byte-compiling %s to %s", file, cfile_base)
                     if not dry_run:
-                        compile(file, cfile, dfile)
+                        compile(file, cfile, dfile, noopt=False)
                 else:
                     log.debug("skipping byte-compilation of %s to %s",
                               file, cfile_base)

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
@@ -290,6 +290,7 @@ def _write_atomic(path, data, mode=0o666):
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
+_NOOPT = 'noopt'
 _OPT = 'opt-'
 
 SOURCE_SUFFIXES = ['.py']  # _setup() adds .pyw as needed.
@@ -298,7 +299,8 @@ def _write_atomic(path, data, mode=0o666):
 # Deprecated.
 DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
 
-def cache_from_source(path, debug_override=None, *, optimization=None):
+def cache_from_source(path, debug_override=None, *, optimization=None,
+                      noopt=None):
     """Given the path to a .py file, return the path to its .pyc file.
 
     The .py file does not need to exist; this simply returns the path to the
@@ -330,16 +332,21 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
     if tag is None:
         raise NotImplementedError('sys.implementation.cache_tag is None')
     almost_filename = ''.join([(base if base else rest), sep, tag])
-    if optimization is None:
-        if sys.flags.optimize == 0:
-            optimization = ''
-        else:
-            optimization = sys.flags.optimize
-    optimization = str(optimization)
-    if optimization != '':
-        if not optimization.isalnum():
-            raise ValueError('{!r} is not alphanumeric'.format(optimization))
-        almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
+    if noopt is None:
+        noopt = sys.flags.noopt
+    if noopt:
+        almost_filename = f'{almost_filename}.{_NOOPT}'
+    else:
+        if optimization is None:
+            if sys.flags.optimize == 0:
+                optimization = ''
+            else:
+                optimization = sys.flags.optimize
+        optimization = str(optimization)
+        if optimization != '':
+            if not optimization.isalnum():
+                raise ValueError('{!r} is not alphanumeric'.format(optimization))
+            almost_filename = f'{almost_filename}.{_OPT}{optimization}'
     filename = almost_filename + BYTECODE_SUFFIXES[0]
     if sys.pycache_prefix is not None:
         # We need an absolute path to the py file to avoid the possibility of
@@ -398,13 +405,14 @@ def source_from_cache(path):
         raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
     elif dot_count == 3:
         optimization = pycache_filename.rsplit('.', 2)[-2]
-        if not optimization.startswith(_OPT):
+        if optimization.startswith(_OPT):
+            opt_level = optimization[len(_OPT):]
+            if not opt_level.isalnum():
+                raise ValueError(f"optimization level {optimization!r} "
+                                 "is not an alphanumeric value")
+        elif not optimization.startswith(_NOOPT):
             raise ValueError("optimization portion of filename does not start "
-                             f"with {_OPT!r}")
-        opt_level = optimization[len(_OPT):]
-        if not opt_level.isalnum():
-            raise ValueError(f"optimization level {optimization!r} is not an "
-                             "alphanumeric value")
+                             f"with {_OPT!r} nor {_NOOPT!r}")
     base_filename = pycache_filename.partition('.')[0]
     return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
 
@@ -845,13 +853,14 @@ def get_source(self, fullname):
                               name=fullname) from exc
         return decode_source(source_bytes)
 
-    def source_to_code(self, data, path, *, _optimize=-1):
+    def source_to_code(self, data, path, *, _optimize=-1, _noopt=None):
         """Return the code object compiled from source.
 
         The 'data' argument can be any object type that compile() supports.
         """
         return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
-                                        dont_inherit=True, optimize=_optimize)
+                                        dont_inherit=True, optimize=_optimize,
+                                        noopt=_noopt)
 
     def get_code(self, fullname):
         """Concrete implementation of InspectLoader.get_code.