From 4c3e538f19ab9678c3a3163d86cff13ade54f0d9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 Nov 2017 00:41:50 +0100 Subject: [PATCH 1/4] bpo-32101: Add sys.flags.dev_mode flag Rename also the "Developer mode" to the "Development mode". --- Doc/library/sys.rst | 4 ++++ Doc/using/cmdline.rst | 2 +- Doc/whatsnew/3.7.rst | 11 ++++++++--- Lib/asyncio/coroutines.py | 2 +- Lib/test/test_cmd_line.py | 18 ++++++++++++++++-- Lib/test/test_sys.py | 6 ++++-- .../2017-11-29-00-42-47.bpo-321010.-axD5l.rst | 1 + Python/sysmodule.c | 5 ++++- 8 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index faf540c4ea43b21..9e47681804b5eb5 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -334,6 +334,7 @@ always available. :const:`bytes_warning` :option:`-b` :const:`quiet` :option:`-q` :const:`hash_randomization` :option:`-R` + :const:`dev_mode` :option:`-X` ``dev`` ============================= ============================= .. versionchanged:: 3.2 @@ -345,6 +346,9 @@ always available. .. versionchanged:: 3.3 Removed obsolete ``division_warning`` attribute. + .. versionchanged:: 3.7 + Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag. + .. data:: float_info diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index e6189fd8127fe55..d26ccc1c3178329 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -411,7 +411,7 @@ Miscellaneous options nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. - * ``-X dev``: enable CPython's "developer mode", introducing additional + * ``-X dev``: enable CPython's "development mode", introducing additional runtime checks which are too expensive to be enabled by default. It should not be more verbose than the default if the code is correct: new warnings are only emitted when an issue is detected. Effect of the developer mode: diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index e4600fe5ffd3169..0cb2f7259b9deda 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -185,10 +185,10 @@ resolution on Linux and Windows. PEP written and implemented by Victor Stinner -New Developer Mode: -X dev --------------------------- +New Development Mode: -X dev +---------------------------- -Add a new "developer mode": ``-X dev`` command line option to enable debug +Add a new "development mode": ``-X dev`` command line option to enable debug checks at runtime. In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W @@ -371,6 +371,11 @@ string expression pattern for braced placeholders and non-braced placeholders separately. (Contributed by Barry Warsaw in :issue:`1198569`.) +sys +--- + +Added :attr:`sys.flags.dev_mode` flag for the new development mode. + time ---- diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 3e305f90abb9fea..642e6e80599fc18 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -31,7 +31,7 @@ def _is_debug_mode(): # when _DEBUG is true. debug = (not sys.flags.ignore_environment and bool(os.environ.get('PYTHONASYNCIODEBUG'))) - if hasattr(sys, '_xoptions') and 'dev' in sys._xoptions: + if sys.flags.dev_mode: debug = True return debug diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 96405e70afc0c11..5ae4ef6633e469f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -508,14 +508,18 @@ def test_sys_flags_set(self): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) - def run_xdev(self, *args, check_exitcode=True): + def run_xdev(self, *args, check_exitcode=True, xdev=True): env = dict(os.environ) env.pop('PYTHONWARNINGS', None) + env.pop('PYTHONDEVMODE', None) # Force malloc() to disable the debug hooks which are enabled # by default for Python compiled in debug mode env['PYTHONMALLOC'] = 'malloc' - args = (sys.executable, '-X', 'dev', *args) + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -526,6 +530,14 @@ def run_xdev(self, *args, check_exitcode=True): return proc.stdout.rstrip() def test_xdev(self): + # sys.flags.dev_mode + code = "import sys; print(sys.flags.dev_mode)" + out = self.run_xdev("-c", code, xdev=False) + self.assertEqual(out, "False") + out = self.run_xdev("-c", code) + self.assertEqual(out, "True") + + # Warnings code = ("import sys, warnings; " "print(' '.join('%s::%s' % (f[0], f[2].__name__) " "for f in warnings.filters))") @@ -555,6 +567,7 @@ def test_xdev(self): "default::ResourceWarning " "default::Warning") + # Memory allocator debug hooks try: import _testcapi except ImportError: @@ -569,6 +582,7 @@ def test_xdev(self): alloc_name = "malloc_debug" self.assertEqual(out, alloc_name) + # Faulthandler try: import faulthandler except ImportError: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 4b8fcb9540ffe58..6346094ad08b4cc 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -526,10 +526,12 @@ def test_sys_flags(self): attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet", "hash_randomization", "isolated") + "bytes_warning", "quiet", "hash_randomization", "isolated", + "dev_mode") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) - self.assertEqual(type(getattr(sys.flags, attr)), int, attr) + attr_type = bool if attr == "dev_mode" else int + self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) self.assertEqual(len(sys.flags), len(attrs)) diff --git a/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst new file mode 100644 index 000000000000000..715a269a5c0a008 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst @@ -0,0 +1 @@ +Add :attr:`sys.flags.dev_mode` flag diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d786739188a8160..64bc14e9c3d7339 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1814,6 +1814,7 @@ static PyStructSequence_Field flags_fields[] = { {"quiet", "-q"}, {"hash_randomization", "-R"}, {"isolated", "-I"}, + {"dev_mode", "-X dev"}, {0} }; @@ -1821,7 +1822,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 13 + 14 }; static PyObject* @@ -1829,6 +1830,7 @@ make_flags(void) { int pos = 0; PyObject *seq; + _PyCoreConfig *core_config = &_PyGILState_GetInterpreterStateUnsafe()->core_config; seq = PyStructSequence_New(&FlagsType); if (seq == NULL) @@ -1853,6 +1855,7 @@ make_flags(void) SetFlag(Py_HashRandomizationFlag); SetFlag(Py_IsolatedFlag); #undef SetFlag + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(core_config->dev_mode)); if (PyErr_Occurred()) { Py_DECREF(seq); From 0eb6e7d7024d2652cd50760443f1fae1309cdb56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 Nov 2017 00:59:29 +0100 Subject: [PATCH 2/4] bpo-32101: Add PYTHONDEVMODE environment variable Mention it in the development chapiter. --- Doc/library/development.rst | 3 +++ Doc/using/cmdline.rst | 10 ++++++++++ Lib/test/test_cmd_line.py | 18 ++++++++++++++++++ Modules/main.c | 7 +++++-- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Doc/library/development.rst b/Doc/library/development.rst index d2b5fa2aa4f5f82..ab34e1f7ce5f64e 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -24,3 +24,6 @@ The list of modules described in this chapter is: unittest.mock-examples.rst 2to3.rst test.rst + +See also the Python development mode: the :option:`-X` ``dev`` option and +:envvar:`PYTHONDEVMODE` environment variable. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index d26ccc1c3178329..d110ae3baf28004 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -426,6 +426,8 @@ Miscellaneous options * Enable the :mod:`faulthandler` module to dump the Python traceback on a crash. * Enable :ref:`asyncio debug mode `. + * Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to + ``True`` It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -796,6 +798,14 @@ conflict. .. versionadded:: 3.7 See :pep:`538` for more details. + +.. envvar:: PYTHONDEVMODE + + If this environment variable is set to a non-empty string, enable the + CPython "development mode". See the :option:`-X` ``dev`` option. + + .. versionadded:: 3.7 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 5ae4ef6633e469f..e4a3052cd556fcb 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -635,6 +635,24 @@ def test_pythonmalloc(self): with self.subTest(env_var=env_var, name=name): self.check_pythonmalloc(env_var, name) + def test_pythondevmode_env(self): + # Test the PYTHONDEVMODE environment variable + code = "import sys; print(sys.flags.dev_mode)" + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + args = (sys.executable, '-c', code) + + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) + + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) + class IgnoreEnvironmentTest(unittest.TestCase): diff --git a/Modules/main.c b/Modules/main.c index ec33b5f086f0f1b..e9d524a1463627c 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -124,7 +124,8 @@ static const char usage_6[] = " hooks.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n"; +" locale coercion and locale compatibility warnings on stderr.\n" +"PYTHONDEVMODE: enable the development mode.\n"; static void pymain_usage(int error, const wchar_t* program) @@ -1520,7 +1521,9 @@ pymain_parse_envvars(_PyMain *pymain) if (pymain_init_tracemalloc(pymain) < 0) { return -1; } - if (pymain_get_xoption(pymain, L"dev")) { + if (pymain_get_xoption(pymain, L"dev" ) || + pymain_get_env_var("PYTHONDEVMODE")) + { core_config->dev_mode = 1; core_config->faulthandler = 1; core_config->allocator = "debug"; From eaaa19c374b340bf21d30e243283492ee45a1ba0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 29 Nov 2017 18:31:38 +0100 Subject: [PATCH 3/4] Fix test_cmd_line when PYTHONMALLOC is set --- Lib/test/test_cmd_line.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index e4a3052cd556fcb..383302bad8b549d 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -595,6 +595,7 @@ def test_xdev(self): def check_pythonmalloc(self, env_var, name): code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) if env_var is not None: env['PYTHONMALLOC'] = env_var else: From f190b4a59482ee3849c9539516afb3a8228e6464 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 30 Nov 2017 11:13:26 +0100 Subject: [PATCH 4/4] asyncio: cleanup _is_debug_mode() code --- Lib/asyncio/coroutines.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 642e6e80599fc18..2b9ea19ed8da498 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -29,11 +29,9 @@ def _is_debug_mode(): # before you define your coroutines. A downside of using this feature # is that tracebacks show entries for the CoroWrapper.__next__ method # when _DEBUG is true. - debug = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - if sys.flags.dev_mode: - debug = True - return debug + return (sys.flags.dev_mode + or (not sys.flags.ignore_environment + and bool(os.environ.get('PYTHONASYNCIODEBUG')))) _DEBUG = _is_debug_mode()