From 111371de1b09caf68576a5a7ded9cc28dbd29f76 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Feb 2019 15:17:10 +0100 Subject: [PATCH 1/6] bpo-36142: Rework Python init to add _PyPreConfig I added a _PyCoreConfig structure to Python 3.7 which contains almost all parameters used to configure Python. Problems: _PyCoreConfig uses bytes and Unicode strings (char* and wchar_t*) whereas it is also used to setup the memory allocator and (filesystem, locale and stdio) encodings. I propose to add a new _PyPreConfig which is the "strict minimum" configuration to setup encodings and the memory allocator. In practice, it also contains parameters which directly or indirectly impacts the allocator and encodings. For example, isolated impacts use_environment which impacts the allocator (PYTHONMALLOC environment variable). Another example: dev_mode=1 sets the allocator to "debug". The command line arguments are now parsed twice. _PyPreConfig only parses a few parameters like -E, -I and -X. A temporary _PyPreCmdline is used to store command line arguments like -X options. I moved structures closer to where they are used. "Global" _PyMain structure has been removed. _PyCmdline now lives way shorter than previously and is moved from main.c to coreconfig.c. The idea is to better control when and how memory is allocated. In term of API, we get something like: _PyCoreConfig config = _PyCoreConfig_INIT; config.preconfig.stdio_encoding = "iso8859-1"; config.preconfig.stdio_errors = "replace"; config.user_site_directory = 0; ... _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); } ... Py_Finalize(); return 0; "config.preconfig.stdio_errors" syntax isn't great, but it's simpler to implement than duplicating all _PyPreConfig fields into _PyCoreConfig. Changes: * Add Python/preconfig.c file * Rename Include/coreconfig.h to Include/cpython/coreconfig.h. * Add Include/internal/pycore_coreconfig.h * Add _PyPreConfig structure: pre-initialization configuration for allocator and encodings. * _PyCoreConfig now has a "_PyPreConfig preconfig;" field * Add _PyArgv structure: helper to pass command line argument in bytes (char*) or Unicode (wchar_t*) * Add exitcode to _PyInitError * Add 'exitcode' field to _PyInitError structure * Add _Py_INIT_EXIT() macro * Rename _Py_FatalInitError() to _Py_ExitInitError() * Add new fields to _PyCoreConfig: * run_command * run_filename * run_module * skip_source_first_line * Add _PyMain_Usage() * _Py_InitializeCore() no longer changes the memory allocator. * Hardcode short and long options in _PyOS_GetOpt(). * Make _PyCoreConfig_xxx() functions internal (Py_BUILD_CORE). * Move _PyCmdline from main.c to coreconfig.c * main.c: Remove _PyMain structure --- Include/Python.h | 2 +- Include/{ => cpython}/coreconfig.h | 213 ++-- Include/cpython/pylifecycle.h | 4 +- Include/cpython/pystate.h | 2 + Include/fileobject.h | 3 + Include/internal/pycore_coreconfig.h | 54 + Include/internal/pycore_getopt.h | 9 +- Include/internal/pycore_pathconfig.h | 2 +- Include/internal/pycore_pymem.h | 6 +- Include/pystate.h | 1 - Lib/test/test_embed.py | 5 + Makefile.pre.in | 4 +- Modules/main.c | 1230 +++------------------ Objects/unicodeobject.c | 16 +- PCbuild/pythoncore.vcxproj | 4 +- PCbuild/pythoncore.vcxproj.filters | 8 +- Programs/_testembed.c | 30 +- Python/coreconfig.c | 1503 +++++++++++++++----------- Python/frozenmain.c | 2 +- Python/getopt.c | 20 +- Python/pathconfig.c | 47 +- Python/preconfig.c | 1065 ++++++++++++++++++ Python/pylifecycle.c | 90 +- Python/pystate.c | 1 + Python/sysmodule.c | 12 +- 25 files changed, 2363 insertions(+), 1970 deletions(-) rename Include/{ => cpython}/coreconfig.h (91%) create mode 100644 Include/internal/pycore_coreconfig.h create mode 100644 Python/preconfig.c diff --git a/Include/Python.h b/Include/Python.h index aa60175f5235f37..55b06aeea9846f8 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -128,7 +128,7 @@ #include "codecs.h" #include "pyerrors.h" -#include "coreconfig.h" +#include "cpython/coreconfig.h" #include "pystate.h" #include "context.h" diff --git a/Include/coreconfig.h b/Include/cpython/coreconfig.h similarity index 91% rename from Include/coreconfig.h rename to Include/cpython/coreconfig.h index 0ed3222a15e142e..72f943b2cf1b19a 100644 --- a/Include/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -1,15 +1,15 @@ #ifndef Py_PYCORECONFIG_H #define Py_PYCORECONFIG_H +#ifndef Py_LIMITED_API #ifdef __cplusplus extern "C" { #endif - -#ifndef Py_LIMITED_API typedef struct { const char *prefix; const char *msg; int user_err; + int exitcode; } _PyInitError; /* Almost all errors causing Python initialization to fail */ @@ -21,48 +21,50 @@ typedef struct { #endif #define _Py_INIT_OK() \ - (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0} + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = -1} #define _Py_INIT_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0, .exitcode = -1} /* Error that can be fixed by the user like invalid input parameter. Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1, .exitcode = -1} #define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") +#define _Py_INIT_EXIT(EXITCODE) \ + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = (EXITCODE)} #define _Py_INIT_FAILED(err) \ - (err.msg != NULL) + (err.msg != NULL || err.exitcode != -1) -#endif /* !defined(Py_LIMITED_API) */ typedef struct { - /* Install signal handlers? Yes by default. */ - int install_signal_handlers; + int argc; + int use_bytes_argv; + char * const * bytes_argv; + wchar_t * const * wchar_argv; +} _PyArgv; + + +typedef struct { + char *allocator; /* Memory allocator: PYTHONMALLOC */ + int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */ /* If greater than 0: use environment variables. Set to 0 by -E command line option. If set to -1 (default), it is set to !Py_IgnoreEnvironmentFlag. */ int use_environment; - int use_hash_seed; /* PYTHONHASHSEED=x */ - unsigned long hash_seed; - - const char *allocator; /* Memory allocator: PYTHONMALLOC */ - int dev_mode; /* PYTHONDEVMODE, -X dev */ + /* If greater than 0, enable isolated mode: sys.path contains + neither the script's directory nor the user's site-packages directory. - /* Enable faulthandler? - Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ - int faulthandler; + Set to 1 by the -I command line option. If set to -1 (default), inherit + Py_IsolatedFlag value. */ + int isolated; - /* Enable tracemalloc? - Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ - int tracemalloc; + /* Enable UTF-8 mode? + Set by -X utf8 command line option and PYTHONUTF8 environment variable. + If set to -1 (default), inherit Py_UTF8Mode value. */ + int utf8_mode; - int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ - int show_ref_count; /* -X showrefcount */ - int show_alloc_count; /* -X showalloccount */ - int dump_refs; /* PYTHONDUMPREFS */ - int malloc_stats; /* PYTHONMALLOCSTATS */ int coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */ int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */ @@ -102,10 +104,81 @@ typedef struct { char *filesystem_encoding; char *filesystem_errors; - /* Enable UTF-8 mode? - Set by -X utf8 command line option and PYTHONUTF8 environment variable. - If set to -1 (default), inherit Py_UTF8Mode value. */ - int utf8_mode; + /* Encoding of sys.stdin, sys.stdout and sys.stderr. + Value set from PYTHONIOENCODING environment variable and + Py_SetStandardStreamEncoding() function. + See also 'stdio_errors' attribute. */ + char *stdio_encoding; + + /* Error handler of sys.stdin and sys.stdout. + Value set from PYTHONIOENCODING environment variable and + Py_SetStandardStreamEncoding() function. + See also 'stdio_encoding' attribute. */ + char *stdio_errors; + +#ifdef MS_WINDOWS + /* If greater than 1, use the "mbcs" encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is + set to a non-empty string. If set to -1 (default), inherit + Py_LegacyWindowsFSEncodingFlag value. + + See PEP 529 for more details. */ + int legacy_windows_fs_encoding; + + /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys + standard streams. + + Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to + a non-empty string. If set to -1 (default), inherit + Py_LegacyWindowsStdioFlag value. + + See PEP 528 for more details. */ + int legacy_windows_stdio; +#endif +} _PyPreConfig; + +#ifdef MS_WINDOWS +# define _PyCoreConfig_WINDOWS_INIT \ + .legacy_windows_fs_encoding = -1, \ + .legacy_windows_stdio = -1, +#else +# define _PyCoreConfig_WINDOWS_INIT +#endif + +#define _PyPreConfig_INIT \ + (_PyPreConfig){ \ + _PyCoreConfig_WINDOWS_INIT \ + .use_environment = -1, \ + .isolated = -1, \ + .utf8_mode = -1, \ + .coerce_c_locale = -1 \ + } + + +typedef struct { + _PyPreConfig preconfig; + + /* Install signal handlers? Yes by default. */ + int install_signal_handlers; + + int use_hash_seed; /* PYTHONHASHSEED=x */ + unsigned long hash_seed; + + /* Enable faulthandler? + Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ + int faulthandler; + + /* Enable tracemalloc? + Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ + int tracemalloc; + + int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ + int show_ref_count; /* -X showrefcount */ + int show_alloc_count; /* -X showalloccount */ + int dump_refs; /* PYTHONDUMPREFS */ + int malloc_stats; /* PYTHONMALLOCSTATS */ wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ @@ -139,13 +212,6 @@ typedef struct { wchar_t *dll_path; /* Windows DLL path */ #endif - /* If greater than 0, enable isolated mode: sys.path contains - neither the script's directory nor the user's site-packages directory. - - Set to 1 by the -I command line option. If set to -1 (default), inherit - Py_IsolatedFlag value. */ - int isolated; - /* If equal to zero, disable the import of the module site and the site-dependent manipulations of sys.path that it entails. Also disable these manipulations if site is explicitly imported later (call @@ -239,39 +305,12 @@ typedef struct { If set to -1 (default), it is set to !Py_UnbufferedStdioFlag. */ int buffered_stdio; - /* Encoding of sys.stdin, sys.stdout and sys.stderr. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_errors' attribute. */ - char *stdio_encoding; + /* Skip the first line of the source: -x command line option */ + int skip_source_first_line; - /* Error handler of sys.stdin and sys.stdout. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_encoding' attribute. */ - char *stdio_errors; - -#ifdef MS_WINDOWS - /* If greater than 1, use the "mbcs" encoding instead of the UTF-8 - encoding for the filesystem encoding. - - Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is - set to a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsFSEncodingFlag value. - - See PEP 529 for more details. */ - int legacy_windows_fs_encoding; - - /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys - standard streams. - - Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to - a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsStdioFlag value. - - See PEP 528 for more details. */ - int legacy_windows_stdio; -#endif + wchar_t *run_command; /* -c command line argument */ + wchar_t *run_module; /* -m command line argument */ + wchar_t *run_filename; /* Trailing command line argument without -c or -m */ /* --- Private fields -------- */ @@ -301,26 +340,15 @@ typedef struct { } _PyCoreConfig; -#ifdef MS_WINDOWS -# define _PyCoreConfig_WINDOWS_INIT \ - .legacy_windows_fs_encoding = -1, \ - .legacy_windows_stdio = -1, -#else -# define _PyCoreConfig_WINDOWS_INIT -#endif - #define _PyCoreConfig_INIT \ (_PyCoreConfig){ \ + .preconfig = _PyPreConfig_INIT, \ .install_signal_handlers = 1, \ - .use_environment = -1, \ .use_hash_seed = -1, \ .faulthandler = -1, \ .tracemalloc = -1, \ - .coerce_c_locale = -1, \ - .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1, \ - .isolated = -1, \ .site_import = -1, \ .bytes_warning = -1, \ .inspect = -1, \ @@ -338,33 +366,12 @@ typedef struct { ._frozen = -1} /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); -PyAPI_FUNC(int) _PyCoreConfig_Copy( - _PyCoreConfig *config, - const _PyCoreConfig *config2); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig( - const _PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config); -PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config); -PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv( - const _PyCoreConfig *config, - const char *name); -PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( - const _PyCoreConfig *config, - wchar_t **dest, - wchar_t *wname, - char *name); - /* Used by _testcapi.get_global_config() and _testcapi.get_core_config() */ PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); -#endif #ifdef __cplusplus } #endif -#endif /* !Py_PYCORECONFIG_H */ +#endif /* !Py_LIMITED_API */ +#endif /* !Py_PYCORECONFIG_H */ diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index a3fdeefde01b40d..e401f80b72c661d 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -39,7 +39,7 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( const _PyCoreConfig *config); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalInitError(_PyInitError err); +PyAPI_FUNC(void) _Py_NO_RETURN _Py_ExitInitError(_PyInitError err); /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. @@ -64,7 +64,7 @@ PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); /* Legacy locale support */ PyAPI_FUNC(void) _Py_CoerceLegacyLocale(int warn); -PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void); +PyAPI_FUNC(int) _Py_LegacyLocaleDetected(const char *ctype_locale); PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); #ifdef __cplusplus diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0da59d96d1f75ee..a354e046eaf6515 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -6,6 +6,8 @@ extern "C" { #endif +#include "coreconfig.h" + /* Placeholders while working on the new configuration API * * See PEP 432 for final anticipated contents diff --git a/Include/fileobject.h b/Include/fileobject.h index 89e8dd6a2850ca9..fed4576e673f8e0 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -27,6 +27,9 @@ PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; #endif PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +#ifdef Py_BUILD_CORE +PyAPI_DATA(int) _Py_HasFileSystemDefaultEncodeErrors; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 PyAPI_DATA(int) Py_UTF8Mode; diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h new file mode 100644 index 000000000000000..3c04402175e6ba8 --- /dev/null +++ b/Include/internal/pycore_coreconfig.h @@ -0,0 +1,54 @@ +#ifndef Py_INTERNAL_CORECONFIG_H +#define Py_INTERNAL_CORECONFIG_H +#ifdef __cplusplus +extern "C" { +#endif + +#if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_BUILTIN) +# error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN defined" +#endif + +/* helpers */ +PyAPI_FUNC(const wchar_t*) _Py_GetXOption(int nxoption, wchar_t **xoptions, wchar_t *name); +PyAPI_FUNC(const wchar_t*) _Py_GetProgramFromArgv(int argc, wchar_t * const *argv); +PyAPI_FUNC(_PyInitError) _PyArgv_Decode(const _PyArgv *args, int *argc_p, wchar_t*** argv_p); + +/* _PyPreConfig */ +PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name); +PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config); +PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2); +PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config); +PyAPI_FUNC(_PyInitError) _PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args); +PyAPI_FUNC(_PyInitError) _PyPreConfig_SetAllocator(_PyPreConfig *config); + +/* _PyCoreConfig */ +PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig); +PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); +PyAPI_FUNC(int) _PyCoreConfig_Copy( + _PyCoreConfig *config, + const _PyCoreConfig *config2); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig( + const _PyCoreConfig *config); +PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config); +PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config); +PyAPI_FUNC(const char*) _PyCoreConfig_GetEnv( + const _PyCoreConfig *config, + const char *name); +PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( + const _PyCoreConfig *config, + wchar_t **dest, + wchar_t *wname, + char *name); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv( + _PyCoreConfig *config, + const _PyArgv *args, + const _PyPreConfig *preconfig); +PyAPI_FUNC(void) _Py_ClearArgcArgv(void); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CORECONFIG_H */ diff --git a/Include/internal/pycore_getopt.h b/Include/internal/pycore_getopt.h index e6f4654a666b6b5..564f545998e2b02 100644 --- a/Include/internal/pycore_getopt.h +++ b/Include/internal/pycore_getopt.h @@ -11,13 +11,6 @@ extern wchar_t *_PyOS_optarg; extern void _PyOS_ResetGetOpt(void); -typedef struct { - const wchar_t *name; - int has_arg; - int val; -} _PyOS_LongOption; - -extern int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, - const _PyOS_LongOption *longopts, int *longindex); +extern int _PyOS_GetOpt(int argc, wchar_t **argv, int *longindex); #endif /* !Py_INTERNAL_PYGETOPT_H */ diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index c0731525ca1002e..894a05e3e895e7c 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -13,7 +13,7 @@ PyAPI_FUNC(void) _Py_wstrlist_clear( wchar_t **list); PyAPI_FUNC(wchar_t**) _Py_wstrlist_copy( int len, - wchar_t **list); + wchar_t * const * list); PyAPI_FUNC(_PyInitError) _Py_wstrlist_append( int *len, wchar_t ***list, diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 1e7da87cd75c0aa..cde2ea6194d8315 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -1,5 +1,5 @@ -#ifndef Py_INTERNAL_MEM_H -#define Py_INTERNAL_MEM_H +#ifndef Py_INTERNAL_PYMEM_H +#define Py_INTERNAL_PYMEM_H #ifdef __cplusplus extern "C" { #endif @@ -158,4 +158,4 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_MEM_H */ +#endif /* !Py_INTERNAL_PYMEM_H */ diff --git a/Include/pystate.h b/Include/pystate.h index a541dc8f00969d7..a79a2e60e210770 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -8,7 +8,6 @@ extern "C" { #endif #include "pythread.h" -#include "coreconfig.h" /* This limitation is for performance and simplicity. If needed it can be removed (with effort). */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d35b9f4d5a2f2fd..6c245ebcd1940b3 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -322,6 +322,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, + 'skip_source_first_line': 0, + 'run_command': None, + 'run_module': None, + 'run_filename': None, + '_install_importlib': 1, '_check_hash_pycs_mode': 'default', '_frozen': 0, diff --git a/Makefile.pre.in b/Makefile.pre.in index 1c536f813f8df16..5ae69480bf16a46 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -357,6 +357,7 @@ PYTHON_OBJS= \ Python/mystrtoul.o \ Python/pathconfig.o \ Python/peephole.o \ + Python/preconfig.o \ Python/pyarena.o \ Python/pyctype.o \ Python/pyfpe.o \ @@ -994,7 +995,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/codecs.h \ $(srcdir)/Include/compile.h \ $(srcdir)/Include/complexobject.h \ - $(srcdir)/Include/coreconfig.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ $(srcdir)/Include/dtoa.h \ @@ -1071,6 +1071,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/Python-ast.h \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/coreconfig.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ @@ -1085,6 +1086,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_coreconfig.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ $(srcdir)/Include/internal/pycore_getopt.h \ $(srcdir)/Include/internal/pycore_gil.h \ diff --git a/Modules/main.c b/Modules/main.c index f6e78890085b6f0..c590ebcf73e5436 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1,14 +1,12 @@ /* Python interpreter main program */ #include "Python.h" -#include "osdefs.h" -#include "pycore_getopt.h" +#include "pycore_coreconfig.h" #include "pycore_pathconfig.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" #include "pycore_pystate.h" -#include #ifdef HAVE_SIGNAL_H #include #endif @@ -35,12 +33,6 @@ # include #endif -#if defined(MS_WINDOWS) -# define PYTHONHOMEHELP "\\python{major}{minor}" -#else -# define PYTHONHOMEHELP "/lib/pythonX.X" -#endif - #define COPYRIGHT \ "Type \"help\", \"copyright\", \"credits\" or \"license\" " \ "for more information." @@ -49,123 +41,6 @@ extern "C" { #endif -#define DECODE_LOCALE_ERR(NAME, LEN) \ - (((LEN) == -2) \ - ? _Py_INIT_USER_ERR("cannot decode " NAME) \ - : _Py_INIT_NO_MEMORY()) - - -#ifdef MS_WINDOWS -#define WCSTOK wcstok_s -#else -#define WCSTOK wcstok -#endif - -/* For Py_GetArgcArgv(); set by main() */ -static wchar_t **orig_argv = NULL; -static int orig_argc = 0; - -/* command line options */ -#define BASE_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" - -#define PROGRAM_OPTS BASE_OPTS - -static const _PyOS_LongOption longoptions[] = { - {L"check-hash-based-pycs", 1, 0}, - {NULL, 0, 0}, -}; - -/* Short usage message (with %s for argv0) */ -static const char usage_line[] = -"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; - -/* Long usage message, split into parts < 512 bytes */ -static const char usage_1[] = "\ -Options and arguments (and corresponding environment variables):\n\ --b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ - and comparing bytes/bytearray with str. (-bb: issue errors)\n\ --B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ --c cmd : program passed in as string (terminates option list)\n\ --d : debug output from parser; also PYTHONDEBUG=x\n\ --E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ --h : print this help message and exit (also --help)\n\ -"; -static const char usage_2[] = "\ --i : inspect interactively after running script; forces a prompt even\n\ - if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ --I : isolate Python from the user's environment (implies -E and -s)\n\ --m mod : run library module as a script (terminates option list)\n\ --O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ - .pyc extension; also PYTHONOPTIMIZE=x\n\ --OO : do -O changes and also discard docstrings; add .opt-2 before\n\ - .pyc extension\n\ --q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ --S : don't imply 'import site' on initialization\n\ -"; -static const char usage_3[] = "\ --u : force the stdout and stderr streams to be unbuffered;\n\ - this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ --v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ - can be supplied multiple times to increase verbosity\n\ --V : print the Python version number and exit (also --version)\n\ - when given twice, print more information about the build\n\ --W arg : warning control; arg is action:message:category:module:lineno\n\ - also PYTHONWARNINGS=arg\n\ --x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ --X opt : set implementation-specific option\n\ ---check-hash-based-pycs always|default|never:\n\ - control how Python invalidates hash-based .pyc files\n\ -"; -static const char usage_4[] = "\ -file : program read from script file\n\ -- : program read from stdin (default; interactive mode if a tty)\n\ -arg ...: arguments passed to program in sys.argv[1:]\n\n\ -Other environment variables:\n\ -PYTHONSTARTUP: file executed on interactive startup (no default)\n\ -PYTHONPATH : '%lc'-separated list of directories prefixed to the\n\ - default module search path. The result is sys.path.\n\ -"; -static const char usage_5[] = -"PYTHONHOME : alternate directory (or %lc).\n" -" The default module search path uses %s.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; -static const char usage_6[] = -"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str, bytes and datetime objects. It can also be\n" -" set to an integer in the range [0,4294967295] to get hash values with a\n" -" predictable seed.\n" -"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" -" hooks.\n" -"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n" -"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of choice.\n" -"PYTHONDEVMODE: enable the development mode.\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n"; - -static void -pymain_usage(int error, const wchar_t* program) -{ - FILE *f = error ? stderr : stdout; - - fprintf(f, usage_line, program); - if (error) - fprintf(f, "Try `python -h' for more information.\n"); - else { - fputs(usage_1, f); - fputs(usage_2, f); - fputs(usage_3, f); - fprintf(f, usage_4, (wint_t)DELIM); - fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP); - fputs(usage_6, f); - } -} - static void pymain_run_interactive_hook(void) @@ -248,6 +123,7 @@ pymain_run_module(const wchar_t *modname, int set_argv0) return 0; } + static PyObject * pymain_get_importer(const wchar_t *filename) { @@ -308,166 +184,15 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf) } -/* Main program */ - -typedef struct { - wchar_t **argv; - int nwarnoption; /* Number of -W command line options */ - wchar_t **warnoptions; /* Command line -W options */ - int nenv_warnoption; /* Number of PYTHONWARNINGS environment variables */ - wchar_t **env_warnoptions; /* PYTHONWARNINGS environment variables */ - int print_help; /* -h, -? options */ - int print_version; /* -V option */ -} _PyCmdline; - -/* Structure used by Py_Main() to pass data to subfunctions */ -typedef struct { - /* Input arguments */ - int argc; - int use_bytes_argv; - char **bytes_argv; - wchar_t **wchar_argv; - - /* Exit status or "exit code": result of pymain_main() */ - int status; - /* Error message if a function failed */ - _PyInitError err; - - /* non-zero is stdin is a TTY or if -i option is used */ - int stdin_is_interactive; - int skip_first_line; /* -x option */ - wchar_t *filename; /* Trailing arg without -c or -m */ - wchar_t *command; /* -c argument */ - wchar_t *module; /* -m argument */ -} _PyMain; - -#define _PyMain_INIT {.err = _Py_INIT_OK()} -/* Note: _PyMain_INIT sets other fields to 0/NULL */ - - /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ -#define RUN_CODE(pymain) \ - (pymain->command != NULL || pymain->filename != NULL \ - || pymain->module != NULL) - - -static wchar_t* -pymain_wstrdup(_PyMain *pymain, const wchar_t *str) -{ - wchar_t *str2 = _PyMem_RawWcsdup(str); - if (str2 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return NULL; - } - return str2; -} - - -static int -pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - assert(cmdline->argv == NULL); - - if (pymain->use_bytes_argv) { - /* +1 for a the NULL terminator */ - size_t size = sizeof(wchar_t*) * (pymain->argc + 1); - wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); - if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - for (int i = 0; i < pymain->argc; i++) { - size_t len; - wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); - if (arg == NULL) { - _Py_wstrlist_clear(i, argv); - pymain->err = DECODE_LOCALE_ERR("command line arguments", - (Py_ssize_t)len); - return -1; - } - argv[i] = arg; - } - argv[pymain->argc] = NULL; - - cmdline->argv = argv; - } - else { - cmdline->argv = pymain->wchar_argv; - } - - wchar_t *program; - if (pymain->argc >= 1 && cmdline->argv != NULL) { - program = cmdline->argv[0]; - } - else { - program = L""; - } - config->program = pymain_wstrdup(pymain, program); - if (config->program == NULL) { - return -1; - } - - return 0; -} +#define RUN_CODE(config) \ + (config->run_command != NULL || config->run_filename != NULL \ + || config->run_module != NULL) static void -pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) -{ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); - cmdline->nwarnoption = 0; - cmdline->warnoptions = NULL; - - _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); - cmdline->nenv_warnoption = 0; - cmdline->env_warnoptions = NULL; - - if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(pymain->argc, cmdline->argv); - } - cmdline->argv = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static void -pymain_clear_pymain(_PyMain *pymain) -{ -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(pymain->filename); - CLEAR(pymain->command); - CLEAR(pymain->module); -#undef CLEAR -} - -static void -pymain_clear_config(_PyCoreConfig *config) -{ - /* Clear core config with the memory allocator - used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _PyCoreConfig_Clear(config); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static void -pymain_free(_PyMain *pymain) +pymain_free(void) { _PyImport_Fini2(); @@ -478,17 +203,7 @@ pymain_free(_PyMain *pymain) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); - /* Force the allocator used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - pymain_clear_pymain(pymain); - - _Py_wstrlist_clear(orig_argc, orig_argv); - orig_argc = 0; - orig_argv = NULL; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _Py_ClearArgcArgv(); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -536,220 +251,6 @@ pymain_sys_path_add_path0(PyInterpreterState *interp, PyObject *path0) } -_PyInitError -_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) -{ - if (*len == INT_MAX) { - /* len+1 would overflow */ - return _Py_INIT_NO_MEMORY(); - } - wchar_t *str2 = _PyMem_RawWcsdup(str); - if (str2 == NULL) { - return _Py_INIT_NO_MEMORY(); - } - - size_t size = (*len + 1) * sizeof(list[0]); - wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); - if (list2 == NULL) { - PyMem_RawFree(str2); - return _Py_INIT_NO_MEMORY(); - } - list2[*len] = str2; - *list = list2; - (*len)++; - return _Py_INIT_OK(); -} - - -static int -pymain_wstrlist_append(_PyMain *pymain, int *len, wchar_t ***list, const wchar_t *str) -{ - _PyInitError err = _Py_wstrlist_append(len, list, str); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; -} - - -/* Parse the command line arguments - Return 0 on success. - Return 1 if parsing failed. - Set pymain->err and return -1 on other errors. */ -static int -pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - _PyOS_ResetGetOpt(); - do { - int longindex = -1; - int c = _PyOS_GetOpt(pymain->argc, cmdline->argv, PROGRAM_OPTS, - longoptions, &longindex); - if (c == EOF) { - break; - } - - if (c == 'c') { - /* -c is the last option; following arguments - that look like options are left for the - command to interpret. */ - size_t len = wcslen(_PyOS_optarg) + 1 + 1; - wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); - if (command == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); - command[len - 2] = '\n'; - command[len - 1] = 0; - pymain->command = command; - break; - } - - if (c == 'm') { - /* -m is the last option; following arguments - that look like options are left for the - module to interpret. */ - pymain->module = pymain_wstrdup(pymain, _PyOS_optarg); - if (pymain->module == NULL) { - return -1; - } - break; - } - - switch (c) { - case 0: - // Handle long option. - assert(longindex == 0); // Only one long option now. - if (!wcscmp(_PyOS_optarg, L"always")) { - config->_check_hash_pycs_mode = "always"; - } else if (!wcscmp(_PyOS_optarg, L"never")) { - config->_check_hash_pycs_mode = "never"; - } else if (!wcscmp(_PyOS_optarg, L"default")) { - config->_check_hash_pycs_mode = "default"; - } else { - fprintf(stderr, "--check-hash-based-pycs must be one of " - "'default', 'always', or 'never'\n"); - return 1; - } - break; - - case 'b': - config->bytes_warning++; - break; - - case 'd': - config->parser_debug++; - break; - - case 'i': - config->inspect++; - config->interactive++; - break; - - case 'I': - config->isolated++; - break; - - /* case 'J': reserved for Jython */ - - case 'O': - config->optimization_level++; - break; - - case 'B': - config->write_bytecode = 0; - break; - - case 's': - config->user_site_directory = 0; - break; - - case 'S': - config->site_import = 0; - break; - - case 'E': - config->use_environment = 0; - break; - - case 't': - /* ignored for backwards compatibility */ - break; - - case 'u': - config->buffered_stdio = 0; - break; - - case 'v': - config->verbose++; - break; - - case 'x': - pymain->skip_first_line = 1; - break; - - case 'h': - case '?': - cmdline->print_help++; - break; - - case 'V': - cmdline->print_version++; - break; - - case 'W': - if (pymain_wstrlist_append(pymain, - &cmdline->nwarnoption, - &cmdline->warnoptions, - _PyOS_optarg) < 0) { - return -1; - } - break; - - case 'X': - if (pymain_wstrlist_append(pymain, - &config->nxoption, - &config->xoptions, - _PyOS_optarg) < 0) { - return -1; - } - break; - - case 'q': - config->quiet++; - break; - - case 'R': - config->use_hash_seed = 0; - break; - - /* This space reserved for other options */ - - default: - /* unknown argument: parsing failed */ - return 1; - } - } while (1); - - if (pymain->command == NULL && pymain->module == NULL - && _PyOS_optind < pymain->argc - && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) - { - pymain->filename = pymain_wstrdup(pymain, cmdline->argv[_PyOS_optind]); - if (pymain->filename == NULL) { - return -1; - } - } - - /* -c and -m options are exclusive */ - assert(!(pymain->command != NULL && pymain->module != NULL)); - - return 0; -} - - static int add_xoption(PyObject *opts, const wchar_t *s) { @@ -783,7 +284,7 @@ add_xoption(PyObject *opts, const wchar_t *s) static PyObject* -config_create_xoptions_dict(const _PyCoreConfig *config) +mainconfig_create_xoptions_dict(const _PyCoreConfig *config) { int nxoption = config->nxoption; wchar_t **xoptions = config->xoptions; @@ -804,132 +305,16 @@ config_create_xoptions_dict(const _PyCoreConfig *config) } -static _PyInitError -config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) -{ - for (int i = 0; i < len; i++) { - _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - options[i]); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - return _Py_INIT_OK(); -} - - -static _PyInitError -config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) -{ - _PyInitError err; - - assert(config->nwarnoption == 0); - - /* The priority order for warnings configuration is (highest precedence - * first): - * - * - the BytesWarning filter, if needed ('-b', '-bb') - * - any '-W' command line options; then - * - the 'PYTHONWARNINGS' environment variable; then - * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then - * - any implicit filters added by _warnings.c/warnings.py - * - * All settings except the last are passed to the warnings module via - * the `sys.warnoptions` list. Since the warnings module works on the basis - * of "the most recently added filter will be checked first", we add - * the lowest precedence entries first so that later entries override them. - */ - - if (config->dev_mode) { - err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - L"default"); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - - err = config_add_warnings_optlist(config, - cmdline->nenv_warnoption, - cmdline->env_warnoptions); - if (_Py_INIT_FAILED(err)) { - return err; - } - - err = config_add_warnings_optlist(config, - cmdline->nwarnoption, - cmdline->warnoptions); - if (_Py_INIT_FAILED(err)) { - return err; - } - - /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c - * don't even try to emit a warning, so we skip setting the filter in that - * case. - */ - if (config->bytes_warning) { - wchar_t *filter; - if (config->bytes_warning> 1) { - filter = L"error::BytesWarning"; - } - else { - filter = L"default::BytesWarning"; - } - err = _Py_wstrlist_append(&config->nwarnoption, - &config->warnoptions, - filter); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - return _Py_INIT_OK(); -} - - -/* Get warning options from PYTHONWARNINGS environment variable. - Return 0 on success. - Set pymain->err and return -1 on error. */ -static _PyInitError -cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, - _PyCmdline *cmdline) +static int +stdin_is_interactive(const _PyCoreConfig *config) { - wchar_t *env; - int res = _PyCoreConfig_GetEnvDup(config, &env, - L"PYTHONWARNINGS", "PYTHONWARNINGS"); - if (res < 0) { - return DECODE_LOCALE_ERR("PYTHONWARNINGS", res); - } - - if (env == NULL) { - return _Py_INIT_OK(); - } - - - wchar_t *warning, *context = NULL; - for (warning = WCSTOK(env, L",", &context); - warning != NULL; - warning = WCSTOK(NULL, L",", &context)) - { - _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, - &cmdline->env_warnoptions, - warning); - if (_Py_INIT_FAILED(err)) { - PyMem_RawFree(env); - return err; - } - } - PyMem_RawFree(env); - return _Py_INIT_OK(); + return (isatty(fileno(stdin)) || config->interactive); } static void -pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) +config_init_stdio(_PyCoreConfig *config) { - pymain->stdin_is_interactive = (isatty(fileno(stdin)) - || config->interactive); - #if defined(MS_WINDOWS) || defined(__CYGWIN__) /* don't translate newlines (\r\n <=> \n) */ _setmode(fileno(stdin), O_BINARY); @@ -965,13 +350,13 @@ pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) static void -pymain_header(_PyMain *pymain, const _PyCoreConfig *config) +pymain_header(const _PyCoreConfig *config) { if (config->quiet) { return; } - if (!config->verbose && (RUN_CODE(pymain) || !pymain->stdin_is_interactive)) { + if (!config->verbose && (RUN_CODE(config) || !stdin_is_interactive(config))) { return; } @@ -982,85 +367,13 @@ pymain_header(_PyMain *pymain, const _PyCoreConfig *config) } -static int -pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) -{ - /* Copy argv to be able to modify it (to force -c/-m) */ - int argc = pymain->argc - _PyOS_optind; - wchar_t **argv; - - if (argc <= 0 || cmdline->argv == NULL) { - /* Ensure at least one (empty) argument is seen */ - static wchar_t *empty_argv[1] = {L""}; - argc = 1; - argv = _Py_wstrlist_copy(1, empty_argv); - } - else { - argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); - } - - if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - wchar_t *arg0 = NULL; - if (pymain->command != NULL) { - /* Force sys.argv[0] = '-c' */ - arg0 = L"-c"; - } - else if (pymain->module != NULL) { - /* Force sys.argv[0] = '-m'*/ - arg0 = L"-m"; - } - if (arg0 != NULL) { - arg0 = _PyMem_RawWcsdup(arg0); - if (arg0 == NULL) { - _Py_wstrlist_clear(argc, argv); - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; - } - - assert(argc >= 1); - PyMem_RawFree(argv[0]); - argv[0] = arg0; - } - - config->argc = argc; - config->argv = argv; - return 0; -} - - -PyObject* -_Py_wstrlist_as_pylist(int len, wchar_t **list) -{ - assert(list != NULL || len < 1); - - PyObject *pylist = PyList_New(len); - if (pylist == NULL) { - return NULL; - } - - for (int i = 0; i < len; i++) { - PyObject *v = PyUnicode_FromWideChar(list[i], -1); - if (v == NULL) { - Py_DECREF(pylist); - return NULL; - } - PyList_SET_ITEM(pylist, i, v); - } - return pylist; -} - - static void -pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) +pymain_import_readline(const _PyCoreConfig *config) { - if (config->isolated) { + if (config->preconfig.isolated) { return; } - if (!config->inspect && RUN_CODE(pymain)) { + if (!config->inspect && RUN_CODE(config)) { return; } if (!isatty(fileno(stdin))) { @@ -1078,7 +391,7 @@ pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) static void -pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_run_startup(_PyCoreConfig *config, PyCompilerFlags *cf) { const char *startup = _PyCoreConfig_GetEnv(config, "PYTHONSTARTUP"); if (startup == NULL) { @@ -1103,10 +416,10 @@ pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) } -static void -pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +static int +pymain_run_file(_PyCoreConfig *config, PyCompilerFlags *cf) { - const wchar_t *filename = pymain->filename; + const wchar_t *filename = config->run_filename; FILE *fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; @@ -1120,11 +433,10 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n", config->program, cfilename, err, strerror(err)); PyMem_RawFree(cfilename_buffer); - pymain->status = 2; - return; + return 2; } - if (pymain->skip_first_line) { + if (config->skip_source_first_line) { int ch; /* Push back first newline so line numbers remain the same */ while ((ch = getc(fp)) != EOF) { @@ -1140,17 +452,15 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", config->program, filename); - pymain->status = 1; fclose(fp); - return; + return 1; } /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; fclose(fp); - return; + return 1; } PyObject *unicode, *bytes = NULL; @@ -1172,34 +482,33 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) /* PyRun_AnyFileExFlags(closeit=1) calls fclose(fp) before running code */ int run = PyRun_AnyFileExFlags(fp, filename_str, 1, cf); Py_XDECREF(bytes); - pymain->status = (run != 0); + return (run != 0); } -static void -pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +static int +pymain_run_stdin(_PyCoreConfig *config, PyCompilerFlags *cf) { - if (pymain->stdin_is_interactive) { + if (stdin_is_interactive(config)) { Py_InspectFlag = 0; /* do exit on SystemExit */ config->inspect = 0; - pymain_run_startup(pymain, config, cf); + pymain_run_startup(config, cf); pymain_run_interactive_hook(); } /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; - return; + return 1; } int run = PyRun_AnyFileExFlags(stdin, "", 0, cf); - pymain->status = (run != 0); + return (run != 0); } static void -pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_repl(int *exitcode, _PyCoreConfig *config, PyCompilerFlags *cf) { /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ @@ -1208,7 +517,7 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) config->inspect = 1; } - if (!(Py_InspectFlag && pymain->stdin_is_interactive && RUN_CODE(pymain))) { + if (!(Py_InspectFlag && stdin_is_interactive(config) && RUN_CODE(config))) { return; } @@ -1217,193 +526,7 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) pymain_run_interactive_hook(); int res = PyRun_AnyFileFlags(stdin, "", cf); - pymain->status = (res != 0); -} - - -/* Parse the command line. - Handle --version and --help options directly. - - Return 1 if Python must exit. - Return 0 on success. - Set pymain->err and return -1 on failure. */ -static int -pymain_parse_cmdline(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - int res = pymain_parse_cmdline_impl(pymain, config, cmdline); - if (res < 0) { - return -1; - } - if (res) { - pymain_usage(1, config->program); - pymain->status = 2; - return 1; - } - - if (pymain->command != NULL || pymain->module != NULL) { - /* Backup _PyOS_optind */ - _PyOS_optind--; - } - - return 0; -} - - -/* Parse command line options and environment variables. - This code must not use Python runtime apart PyMem_Raw memory allocator. - - Return 0 on success. - Return 1 if Python is done and must exit. - Set pymain->err and return -1 on error. */ -static int -pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - _PyInitError err; - - int res = pymain_parse_cmdline(pymain, config, cmdline); - if (res != 0) { - return res; - } - - if (pymain_init_core_argv(pymain, config, cmdline) < 0) { - return -1; - } - - err = _PyCoreConfig_Read(config); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - - if (config->use_environment) { - err = cmdline_init_env_warnoptions(pymain, config, cmdline); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - } - - err = config_init_warnoptions(config, cmdline); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; -} - - -/* Read the configuration and initialize the LC_CTYPE locale: - enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538). */ -static int -pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) -{ - int init_utf8_mode = Py_UTF8Mode; -#ifdef MS_WINDOWS - int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; -#endif - _PyCoreConfig save_config = _PyCoreConfig_INIT; - int res = -1; - int locale_coerced = 0; - int loops = 0; - - if (_PyCoreConfig_Copy(&save_config, config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); - goto done; - } - - /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); - - while (1) { - int utf8_mode = config->utf8_mode; - int encoding_changed = 0; - - /* Watchdog to prevent an infinite loop */ - loops++; - if (loops == 3) { - pymain->err = _Py_INIT_ERR("Encoding changed twice while " - "reading the configuration"); - goto done; - } - - /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend - on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ - Py_UTF8Mode = config->utf8_mode; -#ifdef MS_WINDOWS - Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; -#endif - - if (pymain_init_cmdline_argv(pymain, config, cmdline) < 0) { - goto done; - } - - int conf_res = pymain_read_conf_impl(pymain, config, cmdline); - if (conf_res != 0) { - res = conf_res; - goto done; - } - - /* The legacy C locale assumes ASCII as the default text encoding, which - * causes problems not only for the CPython runtime, but also other - * components like GNU readline. - * - * Accordingly, when the CLI detects it, it attempts to coerce it to a - * more capable UTF-8 based alternative. - * - * See the documentation of the PYTHONCOERCECLOCALE setting for more - * details. - */ - if (config->coerce_c_locale && !locale_coerced) { - locale_coerced = 1; - _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); - encoding_changed = 1; - } - - if (utf8_mode == -1) { - if (config->utf8_mode == 1) { - /* UTF-8 Mode enabled */ - encoding_changed = 1; - } - } - else { - if (config->utf8_mode != utf8_mode) { - encoding_changed = 1; - } - } - - if (!encoding_changed) { - break; - } - - /* Reset the configuration before reading again the configuration, - just keep UTF-8 Mode value. */ - int new_utf8_mode = config->utf8_mode; - int new_coerce_c_locale = config->coerce_c_locale; - if (_PyCoreConfig_Copy(config, &save_config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); - goto done; - } - pymain_clear_cmdline(pymain, cmdline); - pymain_clear_pymain(pymain); - memset(cmdline, 0, sizeof(*cmdline)); - config->utf8_mode = new_utf8_mode; - config->coerce_c_locale = new_coerce_c_locale; - - /* The encoding changed: read again the configuration - with the new encoding */ - } - res = 0; - -done: - _PyCoreConfig_Clear(&save_config); - Py_UTF8Mode = init_utf8_mode ; -#ifdef MS_WINDOWS - Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; -#endif - return res; + *exitcode = (res != 0); } @@ -1548,7 +671,7 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, } if (main_config->xoptions == NULL) { - main_config->xoptions = config_create_xoptions_dict(config); + main_config->xoptions = mainconfig_create_xoptions_dict(config); if (main_config->xoptions == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1604,61 +727,55 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, } -static int -pymain_init_python_main(_PyMain *pymain, _PyCoreConfig *config, - PyInterpreterState *interp) +static _PyInitError +config_init_python_main(PyInterpreterState *interp) { _PyInitError err; _PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT; - err = _PyMainInterpreterConfig_Read(&main_config, config); + err = _PyMainInterpreterConfig_Read(&main_config, &interp->core_config); if (!_Py_INIT_FAILED(err)) { err = _Py_InitializeMainInterpreter(interp, &main_config); } _PyMainInterpreterConfig_Clear(&main_config); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; + return err; } -static int -pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) +static _PyInitError +pymain_run_python(int *exitcode, PyInterpreterState *interp) { - int res = 0; + _PyInitError err; _PyCoreConfig *config = &interp->core_config; PyObject *main_importer_path = NULL; - if (pymain->filename != NULL) { + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be prepended to sys.path. Otherwise, main_importer_path is set to NULL. */ - main_importer_path = pymain_get_importer(pymain->filename); + main_importer_path = pymain_get_importer(config->run_filename); } if (main_importer_path != NULL) { if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } } - else if (!config->isolated) { + else if (!config->preconfig.isolated) { PyObject *path0 = _PyPathConfig_ComputeArgv0(config->argc, config->argv); if (path0 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - res = -1; + err = _Py_INIT_NO_MEMORY(); goto done; } if (pymain_sys_path_add_path0(interp, path0) < 0) { Py_DECREF(path0); - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } Py_DECREF(path0); @@ -1666,118 +783,76 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) PyCompilerFlags cf = {.cf_flags = 0}; - pymain_header(pymain, config); - pymain_import_readline(pymain, config); + pymain_header(config); + pymain_import_readline(config); - if (pymain->command) { - pymain->status = pymain_run_command(pymain->command, &cf); + if (config->run_command) { + *exitcode = pymain_run_command(config->run_command, &cf); } - else if (pymain->module) { - pymain->status = (pymain_run_module(pymain->module, 1) != 0); + else if (config->run_module) { + *exitcode = (pymain_run_module(config->run_module, 1) != 0); } else if (main_importer_path != NULL) { int sts = pymain_run_module(L"__main__", 0); - pymain->status = (sts != 0); + *exitcode = (sts != 0); } - else if (pymain->filename != NULL) { - pymain_run_file(pymain, config, &cf); + else if (config->run_filename != NULL) { + *exitcode = pymain_run_file(config, &cf); } else { - pymain_run_stdin(pymain, config, &cf); + *exitcode = pymain_run_stdin(config, &cf); } - pymain_repl(pymain, config, &cf); + pymain_repl(exitcode, config, &cf); + err = _Py_INIT_OK(); done: Py_XDECREF(main_importer_path); - return res; + return err; } -static int -pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +static _PyInitError +pymain_init_core(const _PyArgv *args, const _PyPreConfig *preconfig, + PyInterpreterState **interp_p) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { - return -1; - } - - int res = pymain_read_conf(pymain, config, cmdline); - if (res < 0) { - return -1; - } - if (res > 0) { - /* --help or --version command: we are done */ - return 1; - } - - if (cmdline->print_help) { - pymain_usage(0, config->program); - return 1; - } + _PyInitError err; + _PyCoreConfig local_config = _PyCoreConfig_INIT; + _PyCoreConfig *config = &local_config; - if (cmdline->print_version) { - printf("Python %s\n", - (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); - return 1; - } + _PyCoreConfig_GetGlobalConfig(config); - /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ - orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); - if (orig_argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + err = _PyCoreConfig_ReadFromArgv(config, args, preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; } - orig_argc = pymain->argc; - return 0; -} - -/* Read the configuration into _PyCoreConfig and _PyMain, initialize the - LC_CTYPE locale and Py_DecodeLocale(). - - Configuration: - - * Command line arguments - * Environment variables - * Py_xxx global configuration variables - - _PyCmdline is a temporary structure used to prioritize these - variables. */ -static int -pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) -{ - /* Force default allocator, since pymain_free() and pymain_clear_config() - must use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#ifdef Py_DEBUG - PyMemAllocatorEx default_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc); -#endif + _PyCoreConfig_SetGlobalConfig(config); - _PyCmdline cmdline; - memset(&cmdline, 0, sizeof(cmdline)); + config_init_stdio(config); - int res = pymain_cmdline_impl(pymain, config, &cmdline); + err = _Py_InitializeCore(interp_p, config); + if (_Py_INIT_FAILED(err)) { + goto done; + } - pymain_clear_cmdline(pymain, &cmdline); + err = config_init_python_main(*interp_p); + if (_Py_INIT_FAILED(err)) { + goto done; + } + err = _Py_INIT_OK(); -#ifdef Py_DEBUG - /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ - PyMemAllocatorEx cur_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc); - assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); -#endif - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; +done: + _PyCoreConfig_Clear(config); + return err; } -static int -pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) +static _PyInitError +pymain_init(const _PyArgv *args, PyInterpreterState **interp_p) { + _PyInitError err; + /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -1787,59 +862,69 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) fedisableexcept(FE_OVERFLOW); #endif - _PyCoreConfig local_config = _PyCoreConfig_INIT; - _PyCoreConfig *config = &local_config; + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } - _PyCoreConfig_GetGlobalConfig(config); + _PyPreConfig local_preconfig = _PyPreConfig_INIT; + _PyPreConfig *preconfig = &local_preconfig; - int cmd_res = pymain_cmdline(pymain, config); - if (cmd_res < 0) { - _Py_FatalInitError(pymain->err); - } - if (cmd_res == 1) { - pymain_clear_config(config); - return 1; + _PyPreConfig_GetGlobalConfig(preconfig); + + err = _PyPreConfig_InitFromArgv(preconfig, args); + if (_Py_INIT_FAILED(err)) { + goto done; } - _PyCoreConfig_SetGlobalConfig(config); + _PyPreConfig_SetGlobalConfig(preconfig); - pymain_init_stdio(pymain, config); + /* Don't change the memory allocator if Py_Initialize() has already been + called. */ + if (preconfig->allocator != NULL && !_PyRuntime.core_initialized) { + err = _PyPreConfig_SetAllocator(preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; + } + } - PyInterpreterState *interp; - pymain->err = _Py_InitializeCore(&interp, config); - if (_Py_INIT_FAILED(pymain->err)) { - _Py_FatalInitError(pymain->err); + err = pymain_init_core(args, preconfig, interp_p); + if (_Py_INIT_FAILED(err)) { + goto done; } - *interp_p = interp; - pymain_clear_config(config); - config = &interp->core_config; + err = _Py_INIT_OK(); - if (pymain_init_python_main(pymain, config, interp) < 0) { - _Py_FatalInitError(pymain->err); - } - return 0; +done: + _PyPreConfig_Clear(preconfig); + return err; } static int -pymain_main(_PyMain *pymain) +pymain_main(const _PyArgv *args) { + _PyInitError err; + PyInterpreterState *interp; - int res = pymain_init(pymain, &interp); - if (res != 1) { - if (pymain_run_python(pymain, interp) < 0) { - _Py_FatalInitError(pymain->err); - } + err = pymain_init(args, &interp); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } - if (Py_FinalizeEx() < 0) { - /* Value unlikely to be confused with a non-error exit status or - other special meaning */ - pymain->status = 120; - } + int exitcode = 0; + err = pymain_run_python(&exitcode, interp); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + exitcode = 120; } - pymain_free(pymain); + pymain_free(); if (_Py_UnhandledKeyboardInterrupt) { /* https://bugs.python.org/issue1054041 - We need to exit via the @@ -1858,52 +943,37 @@ pymain_main(_PyMain *pymain) #ifdef MS_WINDOWS /* cmd.exe detects this, prints ^C, and offers to terminate. */ /* https://msdn.microsoft.com/en-us/library/cc704588.aspx */ - pymain->status = STATUS_CONTROL_C_EXIT; + exitcode = STATUS_CONTROL_C_EXIT; #else - pymain->status = SIGINT + 128; + exitcode = SIGINT + 128; #endif /* !MS_WINDOWS */ } - return pymain->status; + return exitcode; } int Py_Main(int argc, wchar_t **argv) { - _PyMain pymain = _PyMain_INIT; - pymain.use_bytes_argv = 0; - pymain.argc = argc; - pymain.wchar_argv = argv; - - return pymain_main(&pymain); + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 0, + .bytes_argv = NULL, + .wchar_argv = argv}; + return pymain_main(&args); } int _Py_UnixMain(int argc, char **argv) { - _PyMain pymain = _PyMain_INIT; - pymain.use_bytes_argv = 1; - pymain.argc = argc; - pymain.bytes_argv = argv; - - return pymain_main(&pymain); -} - - -/* this is gonna seem *real weird*, but if you put some other code between - Py_Main() and Py_GetArgcArgv() you will need to adjust the test in the - while statement in Misc/gdbinit:ppystack */ - -/* Make the *original* argc/argv available to other modules. - This is rare, but it is needed by the secureware extension. */ - -void -Py_GetArgcArgv(int *argc, wchar_t ***argv) -{ - *argc = orig_argc; - *argv = orig_argv; + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 1, + .bytes_argv = argv, + .wchar_argv = NULL}; + return pymain_main(&args); } #ifdef __cplusplus diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8141ce757412cbf..e059b134dde85ad 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3507,7 +3507,7 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; #if defined(__APPLE__) - return _PyUnicode_AsUTF8String(unicode, config->filesystem_errors); + return _PyUnicode_AsUTF8String(unicode, config->preconfig.filesystem_errors); #else /* Bootstrap check: if the filesystem codec is implemented in Python, we cannot use it to encode and decode filenames before it is loaded. Load @@ -3516,12 +3516,12 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) initialized and the Python codec is loaded. See initfsencoding(). */ if (interp->fscodec_initialized) { return PyUnicode_AsEncodedString(unicode, - config->filesystem_encoding, - config->filesystem_errors); + config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors); } else { return unicode_encode_locale(unicode, - config->filesystem_errors, 0); + config->preconfig.filesystem_errors, 0); } #endif } @@ -3731,7 +3731,7 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; #if defined(__APPLE__) - return PyUnicode_DecodeUTF8Stateful(s, size, config->filesystem_errors, NULL); + return PyUnicode_DecodeUTF8Stateful(s, size, config->preconfig.filesystem_errors, NULL); #else /* Bootstrap check: if the filesystem codec is implemented in Python, we cannot use it to encode and decode filenames before it is loaded. Load @@ -3740,12 +3740,12 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) initialized and the Python codec is loaded. See initfsencoding(). */ if (interp->fscodec_initialized) { return PyUnicode_Decode(s, size, - config->filesystem_encoding, - config->filesystem_errors); + config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors); } else { return unicode_decode_locale(s, size, - config->filesystem_errors, 0); + config->preconfig.filesystem_errors, 0); } #endif } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b1b9944d1e91903..992c84f8cc81eed 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -110,8 +110,8 @@ - + @@ -141,6 +141,7 @@ + @@ -424,6 +425,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 9dbd0669f76d9b6..df1ae5c3d78dfc1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -81,7 +81,7 @@ Include - + Include @@ -174,6 +174,9 @@ Include + + Include + Include @@ -986,6 +989,9 @@ Python + + Python + Python diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e28d94c175e6a5f..81ccc6a53a23c76 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -139,6 +139,9 @@ static int test_forced_io_encoding(void) static int test_pre_initialization_api(void) { + /* the test doesn't support custom memory allocators */ + putenv("PYTHONMALLOC="); + /* Leading "./" ensures getpath.c can still find the standard library */ _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); @@ -235,6 +238,9 @@ static void bpo20891_thread(void *lockp) static int test_bpo20891(void) { + /* the test doesn't support custom memory allocators */ + putenv("PYTHONMALLOC="); + /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread before calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must call PyEval_InitThreads() for us in this case. */ @@ -437,7 +443,7 @@ static int test_init_from_config(void) config.hash_seed = 123; putenv("PYTHONMALLOC=malloc"); - config.allocator = "malloc_debug"; + config.preconfig.allocator = "malloc_debug"; /* dev_mode=1 is tested in test_init_dev_mode() */ @@ -461,7 +467,7 @@ static int test_init_from_config(void) putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; - config.utf8_mode = 1; + config.preconfig.utf8_mode = 1; putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config.pycache_prefix = L"conf_pycache_prefix"; @@ -536,10 +542,10 @@ static int test_init_from_config(void) #ifdef MS_WINDOWS /* Py_SetStandardStreamEncoding() sets Py_LegacyWindowsStdioFlag to 1. Force it to 0 through the config. */ - config.legacy_windows_stdio = 0; + config.preconfig.legacy_windows_stdio = 0; #endif - config.stdio_encoding = "iso8859-1"; - config.stdio_errors = "replace"; + config.preconfig.stdio_encoding = "iso8859-1"; + config.preconfig.stdio_errors = "replace"; putenv("PYTHONNOUSERSITE="); Py_NoUserSiteDirectory = 0; @@ -553,7 +559,7 @@ static int test_init_from_config(void) _PyInitError err = _Py_InitializeFromConfig(&config); /* Don't call _PyCoreConfig_Clear() since all strings are static */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -607,18 +613,18 @@ static int test_init_isolated(void) _PyCoreConfig config = _PyCoreConfig_INIT; /* Set coerce_c_locale and utf8_mode to not depend on the locale */ - config.coerce_c_locale = 0; - config.utf8_mode = 0; + config.preconfig.coerce_c_locale = 0; + config.preconfig.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; Py_IsolatedFlag = 0; - config.isolated = 1; + config.preconfig.isolated = 1; test_init_env_putenvs(); _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -631,11 +637,11 @@ static int test_init_dev_mode(void) _PyCoreConfig config = _PyCoreConfig_INIT; putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); - config.dev_mode = 1; + config.preconfig.dev_mode = 1; config.program_name = L"./_testembed"; _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 2fb4e3fd196ea68..6d2369ca51a18e5 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -1,17 +1,17 @@ #include "Python.h" +#include "osdefs.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" +#include "pycore_getopt.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" #include "pycore_pathconfig.h" #include "pycore_pystate.h" -#include -#ifdef HAVE_LANGINFO_H -# include -#endif -#include /* setlocale() */ -#ifdef HAVE_LANGINFO_H -#include /* nl_langinfo(CODESET) */ +#ifdef MS_WINDOWS +# define WCSTOK wcstok_s +#else +# define WCSTOK wcstok #endif @@ -23,13 +23,6 @@ /* Global configuration variables */ -/* The filesystem encoding is chosen by config_init_fs_encoding(), - see also initfsencoding(). */ -const char *Py_FileSystemDefaultEncoding = NULL; -int Py_HasFileSystemDefaultEncoding = 0; -const char *Py_FileSystemDefaultEncodeErrors = NULL; -static int _Py_HasFileSystemDefaultEncodeErrors = 0; - /* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change stdin and stdout error handler to "surrogateescape". It is equal to -1 by default: unknown, will be set by Py_Main() */ @@ -55,6 +48,136 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ +static wchar_t **orig_argv = NULL; +static int orig_argc = 0; + + +void +_Py_wstrlist_clear(int len, wchar_t **list) +{ + for (int i=0; i < len; i++) { + PyMem_RawFree(list[i]); + } + PyMem_RawFree(list); +} + + +wchar_t** +_Py_wstrlist_copy(int len, wchar_t * const *list) +{ + assert((len > 0 && list != NULL) || len == 0); + size_t size = len * sizeof(list[0]); + wchar_t **list_copy = PyMem_RawMalloc(size); + if (list_copy == NULL) { + return NULL; + } + for (int i=0; i < len; i++) { + wchar_t* arg = _PyMem_RawWcsdup(list[i]); + if (arg == NULL) { + _Py_wstrlist_clear(i, list_copy); + return NULL; + } + list_copy[i] = arg; + } + return list_copy; +} + + +_PyInitError +_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) +{ + if (*len == INT_MAX) { + /* len+1 would overflow */ + return _Py_INIT_NO_MEMORY(); + } + wchar_t *str2 = _PyMem_RawWcsdup(str); + if (str2 == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + size_t size = (*len + 1) * sizeof(list[0]); + wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); + if (list2 == NULL) { + PyMem_RawFree(str2); + return _Py_INIT_NO_MEMORY(); + } + list2[*len] = str2; + *list = list2; + (*len)++; + return _Py_INIT_OK(); +} + + +PyObject* +_Py_wstrlist_as_pylist(int len, wchar_t **list) +{ + assert(list != NULL || len < 1); + + PyObject *pylist = PyList_New(len); + if (pylist == NULL) { + return NULL; + } + + for (int i = 0; i < len; i++) { + PyObject *v = PyUnicode_FromWideChar(list[i], -1); + if (v == NULL) { + Py_DECREF(pylist); + return NULL; + } + PyList_SET_ITEM(pylist, i, v); + } + return pylist; +} + + +void +_Py_ClearArgcArgv(void) +{ + /* Force the allocator used by _Py_SetArgcArgv() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _Py_wstrlist_clear(orig_argc, orig_argv); + orig_argc = 0; + orig_argv = NULL; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +static int +_Py_SetArgcArgv(int argc, wchar_t * const * argv) +{ + /* _Py_SetArgcArgv() is only called once */ + assert(orig_argv == NULL); + + int res = 0; + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + wchar_t **argv_copy = _Py_wstrlist_copy(argc, argv); + if (argv_copy == NULL) { + res = -1; + } + orig_argv = argv_copy; + orig_argc = argc; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; +} + + +/* Make the *original* argc/argv available to other modules. + This is rare, but it is needed by the secureware extension. */ +void +Py_GetArgcArgv(int *argc, wchar_t ***argv) +{ + *argc = orig_argc; + *argv = orig_argv; +} + + PyObject * _Py_GetGlobalVariablesAsDict(void) { @@ -127,164 +250,12 @@ _Py_GetGlobalVariablesAsDict(void) } -void -_Py_wstrlist_clear(int len, wchar_t **list) -{ - for (int i=0; i < len; i++) { - PyMem_RawFree(list[i]); - } - PyMem_RawFree(list); -} - - -wchar_t** -_Py_wstrlist_copy(int len, wchar_t **list) -{ - assert((len > 0 && list != NULL) || len == 0); - size_t size = len * sizeof(list[0]); - wchar_t **list_copy = PyMem_RawMalloc(size); - if (list_copy == NULL) { - return NULL; - } - for (int i=0; i < len; i++) { - wchar_t* arg = _PyMem_RawWcsdup(list[i]); - if (arg == NULL) { - _Py_wstrlist_clear(i, list_copy); - return NULL; - } - list_copy[i] = arg; - } - return list_copy; -} - - -void -_Py_ClearFileSystemEncoding(void) -{ - if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { - PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); - Py_FileSystemDefaultEncoding = NULL; - } - if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) { - PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); - Py_FileSystemDefaultEncodeErrors = NULL; - } -} - - -/* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors - global configuration variables. */ -int -_Py_SetFileSystemEncoding(const char *encoding, const char *errors) -{ - char *encoding2 = _PyMem_RawStrdup(encoding); - if (encoding2 == NULL) { - return -1; - } - - char *errors2 = _PyMem_RawStrdup(errors); - if (errors2 == NULL) { - PyMem_RawFree(encoding2); - return -1; - } - - _Py_ClearFileSystemEncoding(); - - Py_FileSystemDefaultEncoding = encoding2; - Py_HasFileSystemDefaultEncoding = 0; - - Py_FileSystemDefaultEncodeErrors = errors2; - _Py_HasFileSystemDefaultEncodeErrors = 0; - return 0; -} - - -/* Helper to allow an embedding application to override the normal - * mechanism that attempts to figure out an appropriate IO encoding - */ - -static char *_Py_StandardStreamEncoding = NULL; -static char *_Py_StandardStreamErrors = NULL; - -int -Py_SetStandardStreamEncoding(const char *encoding, const char *errors) -{ - if (Py_IsInitialized()) { - /* This is too late to have any effect */ - return -1; - } - - int res = 0; - - /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), - but Py_Initialize() can change the allocator. Use a known allocator - to be able to release the memory later. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* Can't call PyErr_NoMemory() on errors, as Python hasn't been - * initialised yet. - * - * However, the raw memory allocators are initialised appropriately - * as C static variables, so _PyMem_RawStrdup is OK even though - * Py_Initialize hasn't been called yet. - */ - if (encoding) { - _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); - if (!_Py_StandardStreamEncoding) { - res = -2; - goto done; - } - } - if (errors) { - _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); - if (!_Py_StandardStreamErrors) { - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - } - res = -3; - goto done; - } - } -#ifdef MS_WINDOWS - if (_Py_StandardStreamEncoding) { - /* Overriding the stream encoding implies legacy streams */ - Py_LegacyWindowsStdioFlag = 1; - } -#endif - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - return res; -} - - -void -_Py_ClearStandardStreamEncoding(void) -{ - /* Use the same allocator than Py_SetStandardStreamEncoding() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - /* We won't need them anymore. */ - if (_Py_StandardStreamEncoding) { - PyMem_RawFree(_Py_StandardStreamEncoding); - _Py_StandardStreamEncoding = NULL; - } - if (_Py_StandardStreamErrors) { - PyMem_RawFree(_Py_StandardStreamErrors); - _Py_StandardStreamErrors = NULL; - } - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - /* Free memory allocated in config, but don't clear all attributes */ void _PyCoreConfig_Clear(_PyCoreConfig *config) { + _PyPreConfig_Clear(&config->preconfig); + #define CLEAR(ATTR) \ do { \ PyMem_RawFree(ATTR); \ @@ -320,10 +291,9 @@ _PyCoreConfig_Clear(_PyCoreConfig *config) #endif CLEAR(config->base_exec_prefix); - CLEAR(config->filesystem_encoding); - CLEAR(config->filesystem_errors); - CLEAR(config->stdio_encoding); - CLEAR(config->stdio_errors); + CLEAR(config->run_command); + CLEAR(config->run_module); + CLEAR(config->run_filename); #undef CLEAR #undef CLEAR_WSTRLIST } @@ -334,6 +304,10 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); + if (_PyPreConfig_Copy(&config->preconfig, &config2->preconfig) < 0) { + return -1; + } + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ do { \ @@ -365,12 +339,9 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) } while (0) COPY_ATTR(install_signal_handlers); - COPY_ATTR(use_environment); COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); COPY_ATTR(_install_importlib); - COPY_ATTR(allocator); - COPY_ATTR(dev_mode); COPY_ATTR(faulthandler); COPY_ATTR(tracemalloc); COPY_ATTR(import_time); @@ -379,10 +350,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(dump_refs); COPY_ATTR(malloc_stats); - COPY_ATTR(coerce_c_locale); - COPY_ATTR(coerce_c_locale_warn); - COPY_ATTR(utf8_mode); - COPY_WSTR_ATTR(pycache_prefix); COPY_WSTR_ATTR(module_search_path_env); COPY_WSTR_ATTR(home); @@ -403,7 +370,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #endif COPY_WSTR_ATTR(base_exec_prefix); - COPY_ATTR(isolated); COPY_ATTR(site_import); COPY_ATTR(bytes_warning); COPY_ATTR(inspect); @@ -415,14 +381,12 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(quiet); COPY_ATTR(user_site_directory); COPY_ATTR(buffered_stdio); - COPY_STR_ATTR(filesystem_encoding); - COPY_STR_ATTR(filesystem_errors); - COPY_STR_ATTR(stdio_encoding); - COPY_STR_ATTR(stdio_errors); -#ifdef MS_WINDOWS - COPY_ATTR(legacy_windows_fs_encoding); - COPY_ATTR(legacy_windows_stdio); -#endif + + COPY_ATTR(skip_source_first_line); + COPY_WSTR_ATTR(run_command); + COPY_WSTR_ATTR(run_module); + COPY_WSTR_ATTR(run_filename); + COPY_ATTR(_check_hash_pycs_mode); COPY_ATTR(_frozen); @@ -437,19 +401,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) const char* _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name) { - assert(config->use_environment >= 0); - - if (!config->use_environment) { - return NULL; - } - - const char *var = getenv(name); - if (var && var[0] != '\0') { - return var; - } - else { - return NULL; - } + return _PyPreConfig_GetEnv(&config->preconfig, name); } @@ -458,9 +410,9 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, wchar_t **dest, wchar_t *wname, char *name) { - assert(config->use_environment >= 0); + assert(config->preconfig.use_environment >= 0); - if (!config->use_environment) { + if (!config->preconfig.use_environment) { *dest = NULL; return 0; } @@ -513,8 +465,8 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) config->ATTR = !(VALUE); \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); - COPY_FLAG(isolated, Py_IsolatedFlag); + _PyPreConfig_GetGlobalConfig(&config->preconfig); + COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -522,13 +474,8 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) COPY_FLAG(parser_debug, Py_DebugFlag); COPY_FLAG(verbose, Py_VerboseFlag); COPY_FLAG(quiet, Py_QuietFlag); -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); - COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); -#endif COPY_FLAG(_frozen, Py_FrozenFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); COPY_NOT_FLAG(write_bytecode, Py_DontWriteBytecodeFlag); @@ -552,8 +499,8 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) VAR = !config->ATTR; \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); - COPY_FLAG(isolated, Py_IsolatedFlag); + _PyPreConfig_SetGlobalConfig(&config->preconfig); + COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); COPY_FLAG(interactive, Py_InteractiveFlag); @@ -561,13 +508,8 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) COPY_FLAG(parser_debug, Py_DebugFlag); COPY_FLAG(verbose, Py_VerboseFlag); COPY_FLAG(quiet, Py_QuietFlag); -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); - COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); -#endif COPY_FLAG(_frozen, Py_FrozenFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); COPY_NOT_FLAG(write_bytecode, Py_DontWriteBytecodeFlag); @@ -662,6 +604,7 @@ config_init_program_name(_PyCoreConfig *config) return _Py_INIT_OK(); } + static _PyInitError config_init_executable(_PyCoreConfig *config) { @@ -680,26 +623,11 @@ config_init_executable(_PyCoreConfig *config) return _Py_INIT_OK(); } + static const wchar_t* config_get_xoption(const _PyCoreConfig *config, wchar_t *name) { - int nxoption = config->nxoption; - wchar_t **xoptions = config->xoptions; - for (int i=0; i < nxoption; i++) { - wchar_t *option = xoptions[i]; - size_t len; - wchar_t *sep = wcschr(option, L'='); - if (sep != NULL) { - len = (sep - option); - } - else { - len = wcslen(option); - } - if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { - return option; - } - } - return NULL; + return _Py_GetXOption(config->nxoption, config->xoptions, name); } @@ -760,49 +688,6 @@ config_init_hash_seed(_PyCoreConfig *config) } -static _PyInitError -config_init_utf8_mode(_PyCoreConfig *config) -{ - const wchar_t *xopt = config_get_xoption(config, L"utf8"); - if (xopt) { - wchar_t *sep = wcschr(xopt, L'='); - if (sep) { - xopt = sep + 1; - if (wcscmp(xopt, L"1") == 0) { - config->utf8_mode = 1; - } - else if (wcscmp(xopt, L"0") == 0) { - config->utf8_mode = 0; - } - else { - return _Py_INIT_USER_ERR("invalid -X utf8 option value"); - } - } - else { - config->utf8_mode = 1; - } - return _Py_INIT_OK(); - } - - const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONUTF8"); - if (opt) { - if (strcmp(opt, "1") == 0) { - config->utf8_mode = 1; - } - else if (strcmp(opt, "0") == 0) { - config->utf8_mode = 0; - } - else { - return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " - "variable value"); - } - return _Py_INIT_OK(); - } - - return _Py_INIT_OK(); -} - - static int config_str_to_int(const char *str, int *result) { @@ -840,9 +725,9 @@ config_wstr_to_int(const wchar_t *wstr, int *result) static void -get_env_flag(_PyCoreConfig *config, int *flag, const char *name) +get_env_flag(_PyPreConfig *config, int *flag, const char *name) { - const char *var = _PyCoreConfig_GetEnv(config, name); + const char *var = _PyPreConfig_GetEnv(config, name); if (!var) { return; } @@ -860,41 +745,32 @@ get_env_flag(_PyCoreConfig *config, int *flag, const char *name) static _PyInitError config_read_env_vars(_PyCoreConfig *config) { + _PyPreConfig *preconfig = &config->preconfig; + /* Get environment variables */ - get_env_flag(config, &config->parser_debug, "PYTHONDEBUG"); - get_env_flag(config, &config->verbose, "PYTHONVERBOSE"); - get_env_flag(config, &config->optimization_level, "PYTHONOPTIMIZE"); - get_env_flag(config, &config->inspect, "PYTHONINSPECT"); + get_env_flag(preconfig, &config->parser_debug, "PYTHONDEBUG"); + get_env_flag(preconfig, &config->verbose, "PYTHONVERBOSE"); + get_env_flag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); + get_env_flag(preconfig, &config->inspect, "PYTHONINSPECT"); int dont_write_bytecode = 0; - get_env_flag(config, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); + get_env_flag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); if (dont_write_bytecode) { config->write_bytecode = 0; } int no_user_site_directory = 0; - get_env_flag(config, &no_user_site_directory, "PYTHONNOUSERSITE"); + get_env_flag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); if (no_user_site_directory) { config->user_site_directory = 0; } int unbuffered_stdio = 0; - get_env_flag(config, &unbuffered_stdio, "PYTHONUNBUFFERED"); + get_env_flag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); if (unbuffered_stdio) { config->buffered_stdio = 0; } -#ifdef MS_WINDOWS - get_env_flag(config, &config->legacy_windows_fs_encoding, - "PYTHONLEGACYWINDOWSFSENCODING"); - get_env_flag(config, &config->legacy_windows_stdio, - "PYTHONLEGACYWINDOWSSTDIO"); -#endif - - if (config->allocator == NULL) { - config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC"); - } - if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) { config->dump_refs = 1; } @@ -902,23 +778,6 @@ config_read_env_vars(_PyCoreConfig *config) config->malloc_stats = 1; } - const char *env = _PyCoreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); - if (env) { - if (strcmp(env, "0") == 0) { - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 0; - } - } - else if (strcmp(env, "warn") == 0) { - config->coerce_c_locale_warn = 1; - } - else { - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 1; - } - } - } - wchar_t *path; int res = _PyCoreConfig_GetEnvDup(config, &path, L"PYTHONPATH", "PYTHONPATH"); @@ -1034,11 +893,6 @@ config_read_complex_options(_PyCoreConfig *config) || config_get_xoption(config, L"importtime")) { config->import_time = 1; } - if (config_get_xoption(config, L"dev" ) || - _PyCoreConfig_GetEnv(config, "PYTHONDEVMODE")) - { - config->dev_mode = 1; - } _PyInitError err; if (config->tracemalloc < 0) { @@ -1058,301 +912,71 @@ config_read_complex_options(_PyCoreConfig *config) } -static void -config_init_locale(_PyCoreConfig *config) -{ - /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't - imply that the C locale is always coerced. It is only coerced if - if the LC_CTYPE locale is "C". */ - if (config->coerce_c_locale != 0) { - /* The C locale enables the C locale coercion (PEP 538) */ - if (_Py_LegacyLocaleDetected()) { - config->coerce_c_locale = 1; - } - else { - config->coerce_c_locale = 0; - } - } - -#ifndef MS_WINDOWS - if (config->utf8_mode < 0) { - /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - if (ctype_loc != NULL - && (strcmp(ctype_loc, "C") == 0 - || strcmp(ctype_loc, "POSIX") == 0)) - { - config->utf8_mode = 1; - } - } -#endif -} - - -static const char * -get_stdio_errors(const _PyCoreConfig *config) -{ -#ifndef MS_WINDOWS - const char *loc = setlocale(LC_CTYPE, NULL); - if (loc != NULL) { - /* surrogateescape is the default in the legacy C and POSIX locales */ - if (strcmp(loc, "C") == 0 || strcmp(loc, "POSIX") == 0) { - return "surrogateescape"; - } - -#ifdef PY_COERCE_C_LOCALE - /* surrogateescape is the default in locale coercion target locales */ - if (_Py_IsLocaleCoercionTarget(loc)) { - return "surrogateescape"; - } -#endif - } - - return "strict"; -#else - /* On Windows, always use surrogateescape by default */ - return "surrogateescape"; -#endif -} - - -static _PyInitError -get_locale_encoding(char **locale_encoding) -{ -#ifdef MS_WINDOWS - char encoding[20]; - PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); -#elif defined(__ANDROID__) - const char *encoding = "UTF-8"; -#else - const char *encoding = nl_langinfo(CODESET); - if (!encoding || encoding[0] == '\0') { - return _Py_INIT_USER_ERR("failed to get the locale encoding: " - "nl_langinfo(CODESET) failed"); - } -#endif - *locale_encoding = _PyMem_RawStrdup(encoding); - if (*locale_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - return _Py_INIT_OK(); -} - +/* Read configuration settings from standard locations + * + * This function doesn't make any changes to the interpreter state - it + * merely populates any missing configuration settings. This allows an + * embedding application to completely override a config option by + * setting it before calling this function, or else modify the default + * setting before passing the fully populated config to Py_EndInitialization. + * + * More advanced selective initialization tricks are possible by calling + * this function multiple times with various preconfigured settings. + */ static _PyInitError -config_init_stdio_encoding(_PyCoreConfig *config) +_PyCoreConfig_ReadPreConfig(_PyCoreConfig *config) { - /* If Py_SetStandardStreamEncoding() have been called, use these - parameters. */ - if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { - config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); - if (config->stdio_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - - if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { - config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - - if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { - return _Py_INIT_OK(); - } - - /* PYTHONIOENCODING environment variable */ - const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONIOENCODING"); - if (opt) { - char *pythonioencoding = _PyMem_RawStrdup(opt); - if (pythonioencoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - - char *err = strchr(pythonioencoding, ':'); - if (err) { - *err = '\0'; - err++; - if (!err[0]) { - err = NULL; - } - } - - /* Does PYTHONIOENCODING contain an encoding? */ - if (pythonioencoding[0]) { - if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); - if (config->stdio_encoding == NULL) { - PyMem_RawFree(pythonioencoding); - return _Py_INIT_NO_MEMORY(); - } - } - - /* If the encoding is set but not the error handler, - use "strict" error handler by default. - PYTHONIOENCODING=latin1 behaves as - PYTHONIOENCODING=latin1:strict. */ - if (!err) { - err = "strict"; - } - } - - if (config->stdio_errors == NULL && err != NULL) { - config->stdio_errors = _PyMem_RawStrdup(err); - if (config->stdio_errors == NULL) { - PyMem_RawFree(pythonioencoding); - return _Py_INIT_NO_MEMORY(); - } - } + _PyInitError err; + _PyPreConfig local_preconfig = _PyPreConfig_INIT; + _PyPreConfig_GetGlobalConfig(&local_preconfig); - PyMem_RawFree(pythonioencoding); + if (_PyPreConfig_Copy(&local_preconfig, &config->preconfig) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; } - /* UTF-8 Mode uses UTF-8/surrogateescape */ - if (config->utf8_mode) { - if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup("utf-8"); - if (config->stdio_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - if (config->stdio_errors == NULL) { - config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } + err = _PyPreConfig_Read(&local_preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; } - /* Choose the default error handler based on the current locale. */ - if (config->stdio_encoding == NULL) { - _PyInitError err = get_locale_encoding(&config->stdio_encoding); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - if (config->stdio_errors == NULL) { - const char *errors = get_stdio_errors(config); - config->stdio_errors = _PyMem_RawStrdup(errors); - if (config->stdio_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } + if (_PyPreConfig_Copy(&config->preconfig, &local_preconfig) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; } + err = _Py_INIT_OK(); - return _Py_INIT_OK(); +done: + _PyPreConfig_Clear(&local_preconfig); + return err; } - -static _PyInitError -config_init_fs_encoding(_PyCoreConfig *config) +_PyInitError +_PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig) { -#ifdef MS_WINDOWS - if (config->legacy_windows_fs_encoding) { - /* Legacy Windows filesystem encoding: mbcs/replace */ - if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); - if (config->filesystem_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("replace"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } - } - - /* Windows defaults to utf-8/surrogatepass (PEP 529). - - Note: UTF-8 Mode takes the same code path and the Legacy Windows FS - encoding has the priortiy over UTF-8 Mode. */ - if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); - if (config->filesystem_encoding == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } + _PyInitError err; - if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); - } - } -#else - if (config->filesystem_encoding == NULL) { - if (config->utf8_mode) { - /* UTF-8 Mode use: utf-8/surrogateescape */ - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); - /* errors defaults to surrogateescape above */ - } - else if (_Py_GetForceASCII()) { - config->filesystem_encoding = _PyMem_RawStrdup("ascii"); - } - else { - /* macOS and Android use UTF-8, - other platforms use the locale encoding. */ -#if defined(__APPLE__) || defined(__ANDROID__) - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); -#else - _PyInitError err = get_locale_encoding(&config->filesystem_encoding); - if (_Py_INIT_FAILED(err)) { - return err; - } -#endif - } + _PyCoreConfig_GetGlobalConfig(config); - if (config->filesystem_encoding == NULL) { + if (preconfig) { + if (_PyPreConfig_Copy(&config->preconfig, preconfig) < 0) { return _Py_INIT_NO_MEMORY(); } } - - if (config->filesystem_errors == NULL) { - /* by default, use the "surrogateescape" error handler */ - config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); - if (config->filesystem_errors == NULL) { - return _Py_INIT_NO_MEMORY(); + else { + err = _PyCoreConfig_ReadPreConfig(config); + if (_Py_INIT_FAILED(err)) { + return err; } } -#endif - return _Py_INIT_OK(); -} - - -/* Read configuration settings from standard locations - * - * This function doesn't make any changes to the interpreter state - it - * merely populates any missing configuration settings. This allows an - * embedding application to completely override a config option by - * setting it before calling this function, or else modify the default - * setting before passing the fully populated config to Py_EndInitialization. - * - * More advanced selective initialization tricks are possible by calling - * this function multiple times with various preconfigured settings. - */ - -_PyInitError -_PyCoreConfig_Read(_PyCoreConfig *config) -{ - _PyInitError err; - _PyCoreConfig_GetGlobalConfig(config); - assert(config->use_environment >= 0); - if (config->isolated > 0) { - config->use_environment = 0; + if (config->preconfig.isolated > 0) { config->user_site_directory = 0; } -#ifdef MS_WINDOWS - if (config->legacy_windows_fs_encoding) { - config->utf8_mode = 0; - } -#endif - - if (config->use_environment) { + if (config->preconfig.use_environment) { err = config_read_env_vars(config); if (_Py_INIT_FAILED(err)) { return err; @@ -1372,13 +996,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config) return err; } - if (config->utf8_mode < 0) { - err = config_init_utf8_mode(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - if (config->home == NULL) { err = config_init_home(config); if (_Py_INIT_FAILED(err)) { @@ -1400,10 +1017,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } } - if (config->coerce_c_locale != 0 || config->utf8_mode < 0) { - config_init_locale(config); - } - if (config->_install_importlib) { err = _PyCoreConfig_InitPathConfig(config); if (_Py_INIT_FAILED(err)) { @@ -1412,13 +1025,10 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } /* default values */ - if (config->dev_mode) { + if (config->preconfig.dev_mode) { if (config->faulthandler < 0) { config->faulthandler = 1; } - if (config->allocator == NULL) { - config->allocator = "debug"; - } } if (config->use_hash_seed < 0) { config->use_hash_seed = 0; @@ -1430,55 +1040,21 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->tracemalloc < 0) { config->tracemalloc = 0; } - if (config->coerce_c_locale < 0) { - config->coerce_c_locale = 0; - } - if (config->utf8_mode < 0) { - config->utf8_mode = 0; - } if (config->argc < 0) { config->argc = 0; } - if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) { - err = config_init_fs_encoding(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - } - - err = config_init_stdio_encoding(config); - if (_Py_INIT_FAILED(err)) { - return err; - } - - assert(config->coerce_c_locale >= 0); - assert(config->use_environment >= 0); - assert(config->filesystem_encoding != NULL); - assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); - assert(config->stdio_errors != NULL); assert(config->_check_hash_pycs_mode != NULL); return _Py_INIT_OK(); } -PyObject * -_PyCoreConfig_AsDict(const _PyCoreConfig *config) -{ - PyObject *dict, *obj; - - dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - #define SET_ITEM(KEY, EXPR) \ do { \ - obj = (EXPR); \ + PyObject *obj = (EXPR); \ if (obj == NULL) { \ - return NULL; \ + goto fail; \ } \ int res = PyDict_SetItemString(dict, (KEY), obj); \ Py_DECREF(obj); \ @@ -1505,12 +1081,49 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #define SET_ITEM_WSTRLIST(NOPTION, OPTIONS) \ SET_ITEM(#OPTIONS, _Py_wstrlist_as_pylist(config->NOPTION, config->OPTIONS)) - SET_ITEM_INT(install_signal_handlers); +static int +_PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) +{ + SET_ITEM_STR(allocator); + SET_ITEM_INT(dev_mode); SET_ITEM_INT(use_environment); + SET_ITEM_INT(isolated); + SET_ITEM_INT(utf8_mode); + SET_ITEM_INT(coerce_c_locale); + SET_ITEM_INT(coerce_c_locale_warn); + SET_ITEM_STR(filesystem_encoding); + SET_ITEM_STR(filesystem_errors); + SET_ITEM_STR(stdio_encoding); + SET_ITEM_STR(stdio_errors); +#ifdef MS_WINDOWS + SET_ITEM_INT(legacy_windows_fs_encoding); + SET_ITEM_INT(legacy_windows_stdio); +#endif + return 0; + +fail: + return -1; +} + + +PyObject * +_PyCoreConfig_AsDict(const _PyCoreConfig *config) +{ + PyObject *dict; + + dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + + if (_PyPreConfig_AsDict(&config->preconfig, dict) < 0) { + Py_DECREF(dict); + return NULL; + } + + SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); - SET_ITEM_STR(allocator); - SET_ITEM_INT(dev_mode); SET_ITEM_INT(faulthandler); SET_ITEM_INT(tracemalloc); SET_ITEM_INT(import_time); @@ -1518,11 +1131,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(show_alloc_count); SET_ITEM_INT(dump_refs); SET_ITEM_INT(malloc_stats); - SET_ITEM_INT(coerce_c_locale); - SET_ITEM_INT(coerce_c_locale_warn); - SET_ITEM_STR(filesystem_encoding); - SET_ITEM_STR(filesystem_errors); - SET_ITEM_INT(utf8_mode); SET_ITEM_WSTR(pycache_prefix); SET_ITEM_WSTR(program_name); SET_ITEM_WSTRLIST(argc, argv); @@ -1540,7 +1148,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #ifdef MS_WINDOWS SET_ITEM_WSTR(dll_path); #endif - SET_ITEM_INT(isolated); SET_ITEM_INT(site_import); SET_ITEM_INT(bytes_warning); SET_ITEM_INT(inspect); @@ -1552,12 +1159,10 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(quiet); SET_ITEM_INT(user_site_directory); SET_ITEM_INT(buffered_stdio); - SET_ITEM_STR(stdio_encoding); - SET_ITEM_STR(stdio_errors); -#ifdef MS_WINDOWS - SET_ITEM_INT(legacy_windows_fs_encoding); - SET_ITEM_INT(legacy_windows_stdio); -#endif + SET_ITEM_INT(skip_source_first_line); + SET_ITEM_WSTR(run_command); + SET_ITEM_WSTR(run_module); + SET_ITEM_WSTR(run_filename); SET_ITEM_INT(_install_importlib); SET_ITEM_STR(_check_hash_pycs_mode); SET_ITEM_INT(_frozen); @@ -1567,6 +1172,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) fail: Py_DECREF(dict); return NULL; +} #undef FROM_STRING #undef FROM_WSTRING @@ -1576,4 +1182,591 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #undef SET_ITEM_STR #undef SET_ITEM_WSTR #undef SET_ITEM_WSTRLIST + + +typedef struct { + int argc; + wchar_t **argv; + int nwarnoption; /* Number of -W command line options */ + wchar_t **warnoptions; /* Command line -W options */ + int nenv_warnoption; /* Number of PYTHONWARNINGS environment variables */ + wchar_t **env_warnoptions; /* PYTHONWARNINGS environment variables */ + int print_help; /* -h, -? options */ + int print_version; /* -V option */ +} _PyCmdline; + + +static void +cmdline_clear(_PyCmdline *cmdline) +{ + _Py_wstrlist_clear(cmdline->argc, cmdline->argv); + cmdline->argc = 0; + cmdline->argv = NULL; + + _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); + cmdline->nwarnoption = 0; + cmdline->warnoptions = NULL; + + _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); + cmdline->nenv_warnoption = 0; + cmdline->env_warnoptions = NULL; +} + + +#if defined(MS_WINDOWS) +# define PYTHONHOMEHELP "\\python{major}{minor}" +#else +# define PYTHONHOMEHELP "/lib/pythonX.X" +#endif + +/* Short usage message (with %s for argv0) */ +static const char usage_line[] = +"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; + +/* Long usage message, split into parts < 512 bytes */ +static const char usage_1[] = "\ +Options and arguments (and corresponding environment variables):\n\ +-b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ + and comparing bytes/bytearray with str. (-bb: issue errors)\n\ +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ +-c cmd : program passed in as string (terminates option list)\n\ +-d : debug output from parser; also PYTHONDEBUG=x\n\ +-E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ +-h : print this help message and exit (also --help)\n\ +"; +static const char usage_2[] = "\ +-i : inspect interactively after running script; forces a prompt even\n\ + if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ +-I : isolate Python from the user's environment (implies -E and -s)\n\ +-m mod : run library module as a script (terminates option list)\n\ +-O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ + .pyc extension; also PYTHONOPTIMIZE=x\n\ +-OO : do -O changes and also discard docstrings; add .opt-2 before\n\ + .pyc extension\n\ +-q : don't print version and copyright messages on interactive startup\n\ +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ +-S : don't imply 'import site' on initialization\n\ +"; +static const char usage_3[] = "\ +-u : force the stdout and stderr streams to be unbuffered;\n\ + this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ +-v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ + can be supplied multiple times to increase verbosity\n\ +-V : print the Python version number and exit (also --version)\n\ + when given twice, print more information about the build\n\ +-W arg : warning control; arg is action:message:category:module:lineno\n\ + also PYTHONWARNINGS=arg\n\ +-x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ +-X opt : set implementation-specific option\n\ +--check-hash-based-pycs always|default|never:\n\ + control how Python invalidates hash-based .pyc files\n\ +"; +static const char usage_4[] = "\ +file : program read from script file\n\ +- : program read from stdin (default; interactive mode if a tty)\n\ +arg ...: arguments passed to program in sys.argv[1:]\n\n\ +Other environment variables:\n\ +PYTHONSTARTUP: file executed on interactive startup (no default)\n\ +PYTHONPATH : '%lc'-separated list of directories prefixed to the\n\ + default module search path. The result is sys.path.\n\ +"; +static const char usage_5[] = +"PYTHONHOME : alternate directory (or %lc).\n" +" The default module search path uses %s.\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" +"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; +static const char usage_6[] = +"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str, bytes and datetime objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash values with a\n" +" predictable seed.\n" +"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" +" hooks.\n" +"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" +" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" +" locale coercion and locale compatibility warnings on stderr.\n" +"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" +" debugger. It can be set to the callable of your debugger of choice.\n" +"PYTHONDEVMODE: enable the development mode.\n" +"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files.\n"; + + +static void +python_cmdline_usage(const wchar_t* program, int error) +{ + FILE *f = error ? stderr : stdout; + + fprintf(f, usage_line, program); + if (error) + fprintf(f, "Try `python -h' for more information.\n"); + else { + fputs(usage_1, f); + fputs(usage_2, f); + fputs(usage_3, f); + fprintf(f, usage_4, (wint_t)DELIM); + fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP); + fputs(usage_6, f); + } +} + + +/* Parse the command line arguments. */ +static _PyInitError +config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, int *need_usage) +{ + _PyInitError err; + _PyOS_ResetGetOpt(); + do { + int longindex = -1; + int c = _PyOS_GetOpt(cmdline->argc, cmdline->argv, &longindex); + if (c == EOF) { + break; + } + + if (c == 'c') { + /* -c is the last option; following arguments + that look like options are left for the + command to interpret. */ + size_t len = wcslen(_PyOS_optarg) + 1 + 1; + wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); + if (command == NULL) { + return _Py_INIT_NO_MEMORY(); + } + memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); + command[len - 2] = '\n'; + command[len - 1] = 0; + config->run_command = command; + break; + } + + if (c == 'm') { + /* -m is the last option; following arguments + that look like options are left for the + module to interpret. */ + config->run_module = _PyMem_RawWcsdup(_PyOS_optarg); + if (config->run_module == NULL) { + return _Py_INIT_NO_MEMORY(); + } + break; + } + + switch (c) { + case 0: + // Handle long option. + assert(longindex == 0); // Only one long option now. + if (!wcscmp(_PyOS_optarg, L"always")) { + config->_check_hash_pycs_mode = "always"; + } else if (!wcscmp(_PyOS_optarg, L"never")) { + config->_check_hash_pycs_mode = "never"; + } else if (!wcscmp(_PyOS_optarg, L"default")) { + config->_check_hash_pycs_mode = "default"; + } else { + fprintf(stderr, "--check-hash-based-pycs must be one of " + "'default', 'always', or 'never'\n"); + *need_usage = 1; + return _Py_INIT_OK(); + } + break; + + case 'I': + case 'E': + /* arguments parsed by preconfig_parse_cmdline() */ + break; + + case 'b': + config->bytes_warning++; + break; + + case 'd': + config->parser_debug++; + break; + + case 'i': + config->inspect++; + config->interactive++; + break; + + /* case 'J': reserved for Jython */ + + case 'O': + config->optimization_level++; + break; + + case 'B': + config->write_bytecode = 0; + break; + + case 's': + config->user_site_directory = 0; + break; + + case 'S': + config->site_import = 0; + break; + + case 't': + /* ignored for backwards compatibility */ + break; + + case 'u': + config->buffered_stdio = 0; + break; + + case 'v': + config->verbose++; + break; + + case 'x': + config->skip_source_first_line = 1; + break; + + case 'h': + case '?': + cmdline->print_help++; + break; + + case 'V': + cmdline->print_version++; + break; + + case 'W': + err = _Py_wstrlist_append(&cmdline->nwarnoption, + &cmdline->warnoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + + case 'X': + err = _Py_wstrlist_append(&config->nxoption, + &config->xoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + + case 'q': + config->quiet++; + break; + + case 'R': + config->use_hash_seed = 0; + break; + + /* This space reserved for other options */ + + default: + /* unknown argument: parsing failed */ + *need_usage = 1; + return _Py_INIT_OK(); + } + } while (1); + + if (config->run_command == NULL && config->run_module == NULL + && _PyOS_optind < cmdline->argc + && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) + { + config->run_filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]); + if (config->run_filename == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->run_command != NULL || config->run_module != NULL) { + /* Backup _PyOS_optind */ + _PyOS_optind--; + } + + /* -c and -m options are exclusive */ + assert(!(config->run_command != NULL && config->run_module != NULL)); + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_argv(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + /* Copy argv to be able to modify it (to force -c/-m) */ + int argc = cmdline->argc - _PyOS_optind; + wchar_t **argv; + + if (argc <= 0 || cmdline->argv == NULL) { + /* Ensure at least one (empty) argument is seen */ + static wchar_t *empty_argv[1] = {L""}; + argc = 1; + argv = _Py_wstrlist_copy(1, empty_argv); + } + else { + argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); + } + + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + wchar_t *arg0 = NULL; + if (config->run_command != NULL) { + /* Force sys.argv[0] = '-c' */ + arg0 = L"-c"; + } + else if (config->run_module != NULL) { + /* Force sys.argv[0] = '-m'*/ + arg0 = L"-m"; + } + if (arg0 != NULL) { + arg0 = _PyMem_RawWcsdup(arg0); + if (arg0 == NULL) { + _Py_wstrlist_clear(argc, argv); + return _Py_INIT_NO_MEMORY(); + } + + assert(argc >= 1); + PyMem_RawFree(argv[0]); + argv[0] = arg0; + } + + config->argc = argc; + config->argv = argv; + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_program(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + config->program = _PyMem_RawWcsdup(program); + if (config->program == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +/* Get warning options from PYTHONWARNINGS environment variable. + Return 0 on success. + Set pymain->err and return -1 on error. */ +static _PyInitError +cmdline_init_env_warnoptions(const _PyCoreConfig *config, _PyCmdline *cmdline) +{ + wchar_t *env; + int res = _PyCoreConfig_GetEnvDup(config, &env, + L"PYTHONWARNINGS", "PYTHONWARNINGS"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONWARNINGS", res); + } + + if (env == NULL) { + return _Py_INIT_OK(); + } + + + wchar_t *warning, *context = NULL; + for (warning = WCSTOK(env, L",", &context); + warning != NULL; + warning = WCSTOK(NULL, L",", &context)) + { + _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, + &cmdline->env_warnoptions, + warning); + if (_Py_INIT_FAILED(err)) { + PyMem_RawFree(env); + return err; + } + } + PyMem_RawFree(env); + return _Py_INIT_OK(); +} + + +static _PyInitError +config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) +{ + for (int i = 0; i < len; i++) { + _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + options[i]); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) +{ + _PyInitError err; + + assert(config->nwarnoption == 0); + + /* The priority order for warnings configuration is (highest precedence + * first): + * + * - the BytesWarning filter, if needed ('-b', '-bb') + * - any '-W' command line options; then + * - the 'PYTHONWARNINGS' environment variable; then + * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then + * - any implicit filters added by _warnings.c/warnings.py + * + * All settings except the last are passed to the warnings module via + * the `sys.warnoptions` list. Since the warnings module works on the basis + * of "the most recently added filter will be checked first", we add + * the lowest precedence entries first so that later entries override them. + */ + + if (config->preconfig.dev_mode) { + err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + L"default"); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = config_add_warnings_optlist(config, + cmdline->nenv_warnoption, + cmdline->env_warnoptions); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_add_warnings_optlist(config, + cmdline->nwarnoption, + cmdline->warnoptions); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c + * don't even try to emit a warning, so we skip setting the filter in that + * case. + */ + if (config->bytes_warning) { + wchar_t *filter; + if (config->bytes_warning> 1) { + filter = L"error::BytesWarning"; + } + else { + filter = L"default::BytesWarning"; + } + err = _Py_wstrlist_append(&config->nwarnoption, + &config->warnoptions, + filter); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, + const _PyPreConfig *preconfig) +{ + _PyInitError err; + + int need_usage = 0; + err = config_parse_cmdline(config, cmdline, &need_usage); + if (_Py_INIT_FAILED(err)) { + return err; + } + if (need_usage) { + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + python_cmdline_usage(program, 1); + return _Py_INIT_EXIT(2); + } + + if (cmdline->print_help) { + const wchar_t *program = _Py_GetProgramFromArgv(cmdline->argc, cmdline->argv); + python_cmdline_usage(program, 0); + return _Py_INIT_EXIT(0); + } + + if (cmdline->print_version) { + printf("Python %s\n", + (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); + return _Py_INIT_EXIT(0); + } + + err = config_init_argv(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_init_program(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = _PyCoreConfig_Read(config, preconfig); + if (_Py_INIT_FAILED(err)) { + return err; + } + + if (config->preconfig.use_environment) { + err = cmdline_init_env_warnoptions(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = config_init_warnoptions(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + return _Py_INIT_OK(); +} + + +static _PyInitError +config_fromargv(_PyCoreConfig *config, const _PyArgv *args, + const _PyPreConfig *preconfig) +{ + _PyInitError err; + + _PyCmdline cmdline; + memset(&cmdline, 0, sizeof(cmdline)); + + err = _PyArgv_Decode(args, &cmdline.argc, &cmdline.argv); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = config_from_cmdline(config, &cmdline, preconfig); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + if (_Py_SetArgcArgv(cmdline.argc, cmdline.argv)) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + err = _Py_INIT_OK(); + +done: + cmdline_clear(&cmdline); + return err; +} + + +/* Read the configuration into _PyCoreConfig. */ +_PyInitError +_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, + const _PyArgv *args, + const _PyPreConfig *preconfig) +{ + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + return config_fromargv(config, args, preconfig); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 616090965b1351b..6554aa75b038f8a 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -86,7 +86,7 @@ Py_FrozenMain(int argc, char **argv) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } #ifdef MS_WINDOWS diff --git a/Python/getopt.c b/Python/getopt.c index c165a94a2d84ca7..e35a0ea8c8a60c8 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -37,6 +37,21 @@ extern "C" { #endif +/* Python command line options */ +#define OPTIONS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" + +typedef struct { + const wchar_t *name; + int has_arg; + int val; +} _PyOS_LongOption; + +static const _PyOS_LongOption longopts[] = { + {L"check-hash-based-pycs", 1, 0}, + {NULL, 0, 0}, +}; + + int _PyOS_opterr = 1; /* generate error messages */ int _PyOS_optind = 1; /* index into argv array */ wchar_t *_PyOS_optarg = NULL; /* optional argument */ @@ -51,8 +66,7 @@ void _PyOS_ResetGetOpt(void) opt_ptr = L""; } -int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, - const _PyOS_LongOption *longopts, int *longindex) +int _PyOS_GetOpt(int argc, wchar_t **argv, int *longindex) { wchar_t *ptr; wchar_t option; @@ -128,7 +142,7 @@ int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, return '_'; } - if ((ptr = wcschr(optstring, option)) == NULL) { + if ((ptr = wcschr(OPTIONS, option)) == NULL) { if (_PyOS_opterr) fprintf(stderr, "Unknown option: -%c\n", (char)option); return '_'; diff --git a/Python/pathconfig.c b/Python/pathconfig.c index c9bddcf6c674d6c..a1bed77b31d1b63 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -2,6 +2,7 @@ #include "Python.h" #include "osdefs.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_pathconfig.h" #include "pycore_pymem.h" @@ -35,12 +36,6 @@ copy_wstr(wchar_t **dst, const wchar_t *src) static void _PyPathConfig_Clear(_PyPathConfig *config) { - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - #define CLEAR(ATTR) \ do { \ PyMem_RawFree(ATTR); \ @@ -57,8 +52,6 @@ _PyPathConfig_Clear(_PyPathConfig *config) CLEAR(config->home); CLEAR(config->program_name); #undef CLEAR - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -70,37 +63,29 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyInitError err; _PyPathConfig new_config = _PyPathConfig_INIT; - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - /* Calculate program_full_path, prefix, exec_prefix, dll_path (Windows), and module_search_path */ err = _PyPathConfig_Calculate_impl(&new_config, core_config); if (_Py_INIT_FAILED(err)) { - goto err; + goto fail; } /* Copy home and program_name from core_config */ if (copy_wstr(&new_config.home, core_config->home) < 0) { err = _Py_INIT_NO_MEMORY(); - goto err; + goto fail; } if (copy_wstr(&new_config.program_name, core_config->program_name) < 0) { err = _Py_INIT_NO_MEMORY(); - goto err; + goto fail; } _PyPathConfig_Clear(path_config); *path_config = new_config; + return _Py_INIT_OK(); - err = _Py_INIT_OK(); - goto done; - -err: +fail: _PyPathConfig_Clear(&new_config); - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } @@ -148,7 +133,15 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) void _PyPathConfig_ClearGlobal(void) { + /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, + since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be + called before Py_Initialize() which can changes the memory allocator. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyPathConfig_Clear(&_Py_path_config); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -187,9 +180,6 @@ wstrlist_join(wchar_t sep, int count, wchar_t **list) _PyInitError _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) { - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyInitError err; _PyPathConfig path_config = _PyPathConfig_INIT; @@ -234,7 +224,6 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) done: _PyPathConfig_Clear(&path_config); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } @@ -329,7 +318,7 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) #endif if (path_config.isolated != -1) { - config->isolated = path_config.isolated; + config->preconfig.isolated = path_config.isolated; } if (path_config.site_import != -1) { config->site_import = path_config.site_import; @@ -392,7 +381,7 @@ pathconfig_global_init(void) _PyInitError err; _PyCoreConfig config = _PyCoreConfig_INIT; - err = _PyCoreConfig_Read(&config); + err = _PyCoreConfig_Read(&config, NULL); if (_Py_INIT_FAILED(err)) { goto error; } @@ -407,7 +396,7 @@ pathconfig_global_init(void) error: _PyCoreConfig_Clear(&config); - _Py_FatalInitError(err); + _Py_ExitInitError(err); } @@ -417,7 +406,7 @@ void Py_SetPath(const wchar_t *path) { if (path == NULL) { - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_ClearGlobal(); return; } diff --git a/Python/preconfig.c b/Python/preconfig.c new file mode 100644 index 000000000000000..39f6d5ff086875a --- /dev/null +++ b/Python/preconfig.c @@ -0,0 +1,1065 @@ +#include +#include "pycore_fileutils.h" +#include "pycore_getopt.h" +#include "pycore_pylifecycle.h" +#include "pycore_pystate.h" +#include +#ifdef HAVE_LANGINFO_H +# include +#endif + +#define DECODE_LOCALE_ERR(NAME, LEN) \ + (((LEN) == -2) \ + ? _Py_INIT_USER_ERR("cannot decode " NAME) \ + : _Py_INIT_NO_MEMORY()) + + +/* The filesystem encoding is chosen by config_init_fs_encoding(), + see also initfsencoding(). */ +const char *Py_FileSystemDefaultEncoding = NULL; +int Py_HasFileSystemDefaultEncoding = 0; +const char *Py_FileSystemDefaultEncodeErrors = NULL; +int _Py_HasFileSystemDefaultEncodeErrors = 0; + + +void +_Py_ClearFileSystemEncoding(void) +{ + if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { + PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); + Py_FileSystemDefaultEncoding = NULL; + } + if (!_Py_HasFileSystemDefaultEncodeErrors && Py_FileSystemDefaultEncodeErrors) { + PyMem_RawFree((char*)Py_FileSystemDefaultEncodeErrors); + Py_FileSystemDefaultEncodeErrors = NULL; + } +} + + +/* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors + global configuration variables. */ +int +_Py_SetFileSystemEncoding(const char *encoding, const char *errors) +{ + char *encoding2 = _PyMem_RawStrdup(encoding); + if (encoding2 == NULL) { + return -1; + } + + char *errors2 = _PyMem_RawStrdup(errors); + if (errors2 == NULL) { + PyMem_RawFree(encoding2); + return -1; + } + + _Py_ClearFileSystemEncoding(); + + Py_FileSystemDefaultEncoding = encoding2; + Py_HasFileSystemDefaultEncoding = 0; + + Py_FileSystemDefaultEncodeErrors = errors2; + _Py_HasFileSystemDefaultEncodeErrors = 0; + return 0; +} + + +/* Helper to allow an embedding application to override the normal + * mechanism that attempts to figure out an appropriate IO encoding + */ + +static char *_Py_StandardStreamEncoding = NULL; +static char *_Py_StandardStreamErrors = NULL; + + +int +Py_SetStandardStreamEncoding(const char *encoding, const char *errors) +{ + if (Py_IsInitialized()) { + /* This is too late to have any effect */ + return -1; + } + + int res = 0; + + /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(), + but Py_Initialize() can change the allocator. Use a known allocator + to be able to release the memory later. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Can't call PyErr_NoMemory() on errors, as Python hasn't been + * initialised yet. + * + * However, the raw memory allocators are initialised appropriately + * as C static variables, so _PyMem_RawStrdup is OK even though + * Py_Initialize hasn't been called yet. + */ + if (encoding) { + _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); + if (!_Py_StandardStreamEncoding) { + res = -2; + goto done; + } + } + if (errors) { + _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); + if (!_Py_StandardStreamErrors) { + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + } + res = -3; + goto done; + } + } +#ifdef MS_WINDOWS + if (_Py_StandardStreamEncoding) { + /* Overriding the stream encoding implies legacy streams */ + Py_LegacyWindowsStdioFlag = 1; + } +#endif + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + return res; +} + + +void +_Py_ClearStandardStreamEncoding(void) +{ + /* Use the same allocator than Py_SetStandardStreamEncoding() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* We won't need them anymore. */ + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + _Py_StandardStreamEncoding = NULL; + } + if (_Py_StandardStreamErrors) { + PyMem_RawFree(_Py_StandardStreamErrors); + _Py_StandardStreamErrors = NULL; + } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +const wchar_t* +_Py_GetProgramFromArgv(int argc, wchar_t * const *argv) +{ + if (argc >= 1 && argv != NULL) { + return argv[0]; + } + else { + return L""; + } +} + + +const wchar_t* +_Py_GetXOption(int nxoption, wchar_t **xoptions, wchar_t *name) +{ + for (int i=0; i < nxoption; i++) { + wchar_t *option = xoptions[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { + return option; + } + } + return NULL; +} + + +_PyInitError +_PyArgv_Decode(const _PyArgv *args, int *argc_p, wchar_t*** argv_p) +{ + wchar_t** argv; + if (args->use_bytes_argv) { + /* +1 for a the NULL terminator */ + size_t size = sizeof(wchar_t*) * (args->argc + 1); + argv = (wchar_t **)PyMem_RawMalloc(size); + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + for (int i = 0; i < args->argc; i++) { + size_t len; + wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len); + if (arg == NULL) { + _Py_wstrlist_clear(i, argv); + return DECODE_LOCALE_ERR("command line arguments", + (Py_ssize_t)len); + } + argv[i] = arg; + } + argv[args->argc] = NULL; + } + else { + argv = _Py_wstrlist_copy(args->argc, args->wchar_argv); + if (argv == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + *argc_p = args->argc; + *argv_p = argv; + return _Py_INIT_OK(); +} + + +#if !defined(__ANDROID__) +# define _PyPreCmdline_USE_LOCALE +#endif + +typedef struct { + int argc; + wchar_t **argv; + int nxoption; /* Number of -X options */ + wchar_t **xoptions; /* -X options */ +#ifdef _PyPreCmdline_USE_LOCALE + char *ctype_locale; +#endif +} _PyPreCmdline; + + +static void +precmdline_clear(_PyPreCmdline *cmdline) +{ + _Py_wstrlist_clear(cmdline->argc, cmdline->argv); + cmdline->argc = 0; + cmdline->argv = NULL; + + _Py_wstrlist_clear(cmdline->nxoption, cmdline->xoptions); + cmdline->nxoption = 0; + cmdline->xoptions = NULL; + +#ifdef _PyPreCmdline_USE_LOCALE + PyMem_RawFree(cmdline->ctype_locale); + cmdline->ctype_locale = NULL; +#endif +} + + +static _PyInitError +precmdline_init(const _PyArgv *args, _PyPreCmdline *cmdline) +{ + _PyInitError err = _PyArgv_Decode(args, &cmdline->argc, &cmdline->argv); + if (_Py_INIT_FAILED(err)) { + return err; + } + +#ifdef _PyPreCmdline_USE_LOCALE + const char *loc = setlocale(LC_CTYPE, NULL); + if (loc != NULL) { + cmdline->ctype_locale = _PyMem_RawStrdup(loc); + if (cmdline->ctype_locale == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#endif + + return _Py_INIT_OK(); +} + + +void +_PyPreConfig_Clear(_PyPreConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->allocator); + CLEAR(config->filesystem_encoding); + CLEAR(config->filesystem_errors); + CLEAR(config->stdio_encoding); + CLEAR(config->stdio_errors); +#undef CLEAR +} + + +int +_PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) +{ + _PyPreConfig_Clear(config); + +#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR +#define COPY_STR_ATTR(ATTR) \ + do { \ + if (config2->ATTR != NULL) { \ + config->ATTR = _PyMem_RawStrdup(config2->ATTR); \ + if (config->ATTR == NULL) { \ + return -1; \ + } \ + } \ + } while (0) + + COPY_ATTR(use_environment); + COPY_ATTR(isolated); + COPY_ATTR(dev_mode); + COPY_STR_ATTR(allocator); + COPY_ATTR(utf8_mode); + COPY_ATTR(coerce_c_locale); + COPY_ATTR(coerce_c_locale_warn); + COPY_STR_ATTR(filesystem_encoding); + COPY_STR_ATTR(filesystem_errors); + COPY_STR_ATTR(stdio_encoding); + COPY_STR_ATTR(stdio_errors); +#ifdef MS_WINDOWS + COPY_ATTR(legacy_windows_fs_encoding); + COPY_ATTR(legacy_windows_stdio); +#endif + return 0; + +#undef COPY_ATTR +#undef COPY_STR_ATTR +} + + +void +_PyPreConfig_GetGlobalConfig(_PyPreConfig *config) +{ +#define COPY_FLAG(ATTR, VALUE) \ + if (config->ATTR == -1) { \ + config->ATTR = VALUE; \ + } +#define COPY_NOT_FLAG(ATTR, VALUE) \ + if (config->ATTR == -1) { \ + config->ATTR = !(VALUE); \ + } + + COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(isolated, Py_IsolatedFlag); +#ifdef MS_WINDOWS + COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); + COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); +#endif + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); + +#undef COPY_FLAG +#undef COPY_NOT_FLAG +} + + +void +_PyPreConfig_SetGlobalConfig(const _PyPreConfig *config) +{ +#define COPY_FLAG(ATTR, VAR) \ + if (config->ATTR != -1) { \ + VAR = config->ATTR; \ + } +#define COPY_NOT_FLAG(ATTR, VAR) \ + if (config->ATTR != -1) { \ + VAR = !config->ATTR; \ + } + + COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(isolated, Py_IsolatedFlag); +#ifdef MS_WINDOWS + COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); + COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); +#endif + COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); + +#undef COPY_FLAG +#undef COPY_NOT_FLAG +} + + +const char* +_PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name) +{ + assert(config->use_environment >= 0); + + if (!config->use_environment) { + return NULL; + } + + const char *var = getenv(name); + if (var && var[0] != '\0') { + return var; + } + else { + return NULL; + } +} + + +static void +preconfig_init_locale(_PyPreConfig *config, const char *ctype_locale) +{ + /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't + imply that the C locale is always coerced. It is only coerced if + if the LC_CTYPE locale is "C". */ + if (config->coerce_c_locale != 0) { + /* The C locale enables the C locale coercion (PEP 538) */ + if (_Py_LegacyLocaleDetected(ctype_locale)) { + config->coerce_c_locale = 1; + } + else { + config->coerce_c_locale = 0; + } + } + +#ifndef MS_WINDOWS + if (config->utf8_mode < 0) { + /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ + if (ctype_locale != NULL + && (strcmp(ctype_locale, "C") == 0 + || strcmp(ctype_locale, "POSIX") == 0)) + { + config->utf8_mode = 1; + } + } +#endif +} + + +static _PyInitError +preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline) +{ + const wchar_t *xopt; + if (cmdline) { + xopt = _Py_GetXOption(cmdline->nxoption, cmdline->xoptions, L"utf8"); + } + else { + xopt = NULL; + } + if (xopt) { + wchar_t *sep = wcschr(xopt, L'='); + if (sep) { + xopt = sep + 1; + if (wcscmp(xopt, L"1") == 0) { + config->utf8_mode = 1; + } + else if (wcscmp(xopt, L"0") == 0) { + config->utf8_mode = 0; + } + else { + return _Py_INIT_USER_ERR("invalid -X utf8 option value"); + } + } + else { + config->utf8_mode = 1; + } + return _Py_INIT_OK(); + } + + const char *opt = _PyPreConfig_GetEnv(config, "PYTHONUTF8"); + if (opt) { + if (strcmp(opt, "1") == 0) { + config->utf8_mode = 1; + } + else if (strcmp(opt, "0") == 0) { + config->utf8_mode = 0; + } + else { + return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " + "variable value"); + } + return _Py_INIT_OK(); + } + + return _Py_INIT_OK(); +} + + +static _PyInitError +get_locale_encoding(char **locale_encoding) +{ +#ifdef MS_WINDOWS + char encoding[20]; + PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP()); +#elif defined(__ANDROID__) + const char *encoding = "UTF-8"; +#else + const char *encoding = nl_langinfo(CODESET); + if (!encoding || encoding[0] == '\0') { + return _Py_INIT_USER_ERR("failed to get the locale encoding: " + "nl_langinfo(CODESET) failed"); + } +#endif + *locale_encoding = _PyMem_RawStrdup(encoding); + if (*locale_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +preconfig_init_fs_encoding(_PyPreConfig *config) +{ +#ifdef MS_WINDOWS + if (config->legacy_windows_fs_encoding) { + /* Legacy Windows filesystem encoding: mbcs/replace */ + if (config->filesystem_encoding == NULL) { + config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + if (config->filesystem_errors == NULL) { + config->filesystem_errors = _PyMem_RawStrdup("replace"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + /* Windows defaults to utf-8/surrogatepass (PEP 529). + + Note: UTF-8 Mode takes the same code path and the Legacy Windows FS + encoding has the priortiy over UTF-8 Mode. */ + if (config->filesystem_encoding == NULL) { + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->filesystem_errors == NULL) { + config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#else + if (config->filesystem_encoding == NULL) { + if (config->utf8_mode) { + /* UTF-8 Mode use: utf-8/surrogateescape */ + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + /* errors defaults to surrogateescape above */ + } + else if (_Py_GetForceASCII()) { + config->filesystem_encoding = _PyMem_RawStrdup("ascii"); + } + else { + /* macOS and Android use UTF-8, + other platforms use the locale encoding. */ +#if defined(__APPLE__) || defined(__ANDROID__) + config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); +#else + _PyInitError err = get_locale_encoding(&config->filesystem_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } +#endif + } + + if (config->filesystem_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->filesystem_errors == NULL) { + /* by default, use the "surrogateescape" error handler */ + config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); + if (config->filesystem_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } +#endif + return _Py_INIT_OK(); +} + + +static const char * +get_stdio_errors(void) +{ +#ifndef MS_WINDOWS + const char *loc = setlocale(LC_CTYPE, NULL); + if (loc != NULL) { + /* surrogateescape is the default in the legacy C and POSIX locales */ + if (strcmp(loc, "C") == 0 || strcmp(loc, "POSIX") == 0) { + return "surrogateescape"; + } + +#ifdef PY_COERCE_C_LOCALE + /* surrogateescape is the default in locale coercion target locales */ + if (_Py_IsLocaleCoercionTarget(loc)) { + return "surrogateescape"; + } +#endif + } + + return "strict"; +#else + /* On Windows, always use surrogateescape by default */ + return "surrogateescape"; +#endif +} + + +static _PyInitError +preconfig_init_stdio_encoding(_PyPreConfig *config) +{ + /* If Py_SetStandardStreamEncoding() have been called, use these + parameters. */ + if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { + config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); + if (config->stdio_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { + config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { + return _Py_INIT_OK(); + } + + /* PYTHONIOENCODING environment variable */ + const char *opt = _PyPreConfig_GetEnv(config, "PYTHONIOENCODING"); + if (opt) { + char *pythonioencoding = _PyMem_RawStrdup(opt); + if (pythonioencoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + char *err = strchr(pythonioencoding, ':'); + if (err) { + *err = '\0'; + err++; + if (!err[0]) { + err = NULL; + } + } + + /* Does PYTHONIOENCODING contain an encoding? */ + if (pythonioencoding[0]) { + if (config->stdio_encoding == NULL) { + config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); + if (config->stdio_encoding == NULL) { + PyMem_RawFree(pythonioencoding); + return _Py_INIT_NO_MEMORY(); + } + } + + /* If the encoding is set but not the error handler, + use "strict" error handler by default. + PYTHONIOENCODING=latin1 behaves as + PYTHONIOENCODING=latin1:strict. */ + if (!err) { + err = "strict"; + } + } + + if (config->stdio_errors == NULL && err != NULL) { + config->stdio_errors = _PyMem_RawStrdup(err); + if (config->stdio_errors == NULL) { + PyMem_RawFree(pythonioencoding); + return _Py_INIT_NO_MEMORY(); + } + } + + PyMem_RawFree(pythonioencoding); + } + + /* UTF-8 Mode uses UTF-8/surrogateescape */ + if (config->utf8_mode) { + if (config->stdio_encoding == NULL) { + config->stdio_encoding = _PyMem_RawStrdup("utf-8"); + if (config->stdio_encoding == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + if (config->stdio_errors == NULL) { + config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + /* Choose the default error handler based on the current locale. */ + if (config->stdio_encoding == NULL) { + _PyInitError err = get_locale_encoding(&config->stdio_encoding); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + if (config->stdio_errors == NULL) { + const char *errors = get_stdio_errors(); + config->stdio_errors = _PyMem_RawStrdup(errors); + if (config->stdio_errors == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + return _Py_INIT_OK(); +} + + +static _PyInitError +preconfig_read_env_vars(_PyPreConfig *config) +{ + if (config->allocator == NULL) { + const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC"); + if (allocator) { + config->allocator = _PyMem_RawStrdup(allocator); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + +#ifdef MS_WINDOWS + get_env_flag(config, &config->legacy_windows_fs_encoding, + "PYTHONLEGACYWINDOWSFSENCODING"); + get_env_flag(config, &config->legacy_windows_stdio, + "PYTHONLEGACYWINDOWSSTDIO"); +#endif + + const char *env = _PyPreConfig_GetEnv(config, "PYTHONCOERCECLOCALE"); + if (env) { + if (strcmp(env, "0") == 0) { + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 0; + } + } + else if (strcmp(env, "warn") == 0) { + config->coerce_c_locale_warn = 1; + } + else { + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 1; + } + } + } + + return _Py_INIT_OK(); +} + + +static _PyInitError +preconfig_read_from_cmdline(_PyPreConfig *config, const _PyPreCmdline *cmdline) +{ + assert(config->use_environment >= 0); + + _PyPreConfig_GetGlobalConfig(config); + + _PyInitError err; + + if (config->isolated > 0) { + config->use_environment = 0; + } + +#ifdef MS_WINDOWS + if (config->legacy_windows_fs_encoding) { + config->utf8_mode = 0; + } +#endif + + if (config->use_environment) { + err = preconfig_read_env_vars(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + if ((cmdline != NULL && _Py_GetXOption(cmdline->nxoption, cmdline->xoptions, L"dev")) || + _PyPreConfig_GetEnv(config, "PYTHONDEVMODE")) + { + config->dev_mode = 1; + } + + if (config->utf8_mode < 0) { + err = preconfig_init_utf8_mode(config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + if (config->coerce_c_locale != 0 || config->utf8_mode < 0) { + const char *ctype_locale; + if (cmdline) { + ctype_locale = cmdline->ctype_locale; + } + else { + ctype_locale = setlocale(LC_CTYPE, NULL); + } + preconfig_init_locale(config, ctype_locale); + } + + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 0; + } + if (config->utf8_mode < 0) { + config->utf8_mode = 0; + } + + if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) { + err = preconfig_init_fs_encoding(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + + err = preconfig_init_stdio_encoding(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* default values */ + if (config->dev_mode) { + if (config->allocator == NULL) { + config->allocator = _PyMem_RawStrdup("debug"); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } + + assert(config->coerce_c_locale >= 0); + assert(config->use_environment >= 0); + assert(config->filesystem_encoding != NULL); + assert(config->filesystem_errors != NULL); + assert(config->stdio_encoding != NULL); + assert(config->stdio_errors != NULL); + + return _Py_INIT_OK(); +} + + +_PyInitError +_PyPreConfig_Read(_PyPreConfig *config) +{ + return preconfig_read_from_cmdline(config, NULL); +} + + +/* Parse _PyPreConfig command line options like -E */ +static _PyInitError +preconfig_parse_cmdline(_PyPreConfig *config, _PyPreCmdline *cmdline) +{ + _PyOS_ResetGetOpt(); + _PyOS_opterr = 0; + do { + int longindex = -1; + int c = _PyOS_GetOpt(cmdline->argc, cmdline->argv, &longindex); + if (c == EOF || c == 'c' || c == 'm') { + break; + } + + switch (c) { + case 'I': + config->isolated++; + break; + + case 'E': + config->use_environment = 0; + break; + + case 'X': + { + _PyInitError err; + err = _Py_wstrlist_append(&cmdline->nxoption, + &cmdline->xoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; + } + break; + } + + default: + /* other arguments will be parsed by + cmdline_parse_impl() */ + break; + } + } while (1); + return _Py_INIT_OK(); +} + + +static _PyInitError +_PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args) +{ + _PyInitError err; + + _PyPreCmdline cmdline; + memset(&cmdline, 0, sizeof(cmdline)); + + err = precmdline_init(args, &cmdline); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = preconfig_parse_cmdline(config, &cmdline); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = preconfig_read_from_cmdline(config, &cmdline); + +done: + precmdline_clear(&cmdline); + return err; +} + + +/* Read the preconfiguration and initialize the LC_CTYPE locale: + coerce the C locale (PEP 538). */ +_PyInitError +_PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args) +{ + _PyInitError err; + + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + int init_utf8_mode = Py_UTF8Mode; +#ifdef MS_WINDOWS + int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; +#endif + _PyPreConfig save_config = _PyPreConfig_INIT; + int locale_coerced = 0; + int loops = 0; + + if (_PyPreConfig_Copy(&save_config, config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + /* Set LC_CTYPE to the user preferred locale */ + _Py_SetLocaleFromEnv(LC_CTYPE); + + while (1) { + int utf8_mode = config->utf8_mode; + + /* Watchdog to prevent an infinite loop */ + loops++; + if (loops == 3) { + err = _Py_INIT_ERR("Encoding changed twice while " + "reading the configuration"); + goto done; + } + + /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend + on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ + Py_UTF8Mode = config->utf8_mode; +#ifdef MS_WINDOWS + Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; +#endif + + err = _PyPreConfig_ReadFromArgv(config, args); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + /* The legacy C locale assumes ASCII as the default text encoding, which + * causes problems not only for the CPython runtime, but also other + * components like GNU readline. + * + * Accordingly, when the CLI detects it, it attempts to coerce it to a + * more capable UTF-8 based alternative. + * + * See the documentation of the PYTHONCOERCECLOCALE setting for more + * details. + */ + int encoding_changed = 0; + if (config->coerce_c_locale && !locale_coerced) { + locale_coerced = 1; + _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); + encoding_changed = 1; + } + + if (utf8_mode == -1) { + if (config->utf8_mode == 1) { + /* UTF-8 Mode enabled */ + encoding_changed = 1; + } + } + else { + if (config->utf8_mode != utf8_mode) { + encoding_changed = 1; + } + } + + if (!encoding_changed) { + break; + } + + /* Reset the configuration before reading again the configuration, + just keep UTF-8 Mode value. */ + int new_utf8_mode = config->utf8_mode; + int new_coerce_c_locale = config->coerce_c_locale; + if (_PyPreConfig_Copy(config, &save_config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + config->utf8_mode = new_utf8_mode; + config->coerce_c_locale = new_coerce_c_locale; + + /* The encoding changed: read again the configuration + with the new encoding */ + } + err = _Py_INIT_OK(); + +done: + _PyPreConfig_Clear(&save_config); + Py_UTF8Mode = init_utf8_mode ; +#ifdef MS_WINDOWS + Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; +#endif + return err; +} + + +_PyInitError +_PyPreConfig_SetAllocator(_PyPreConfig *config) +{ + assert(!_PyRuntime.core_initialized); + + PyMemAllocatorEx old_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_PyMem_SetupAllocators(config->allocator) < 0) { + return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); + } + + /* free config with the old allocator and re-allocate the config + with the new allocator */ + PyMemAllocatorEx new_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &new_alloc); + + _PyInitError err; + _PyPreConfig config2 = _PyPreConfig_INIT; + if (_PyPreConfig_Copy(&config2, config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyPreConfig_Clear(config); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &new_alloc); + + if (_PyPreConfig_Copy(config, &config2) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + err = _Py_INIT_OK(); + +done: + _PyPreConfig_Clear(&config2); + return err; +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index eb0fdddb4b3acf3..80d6c3952143c77 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -5,6 +5,7 @@ #include "Python-ast.h" #undef Yield /* undefine macro conflicting with */ #include "pycore_context.h" +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_hamt.h" #include "pycore_pathconfig.h" @@ -261,7 +262,7 @@ initexternalimport(PyInterpreterState *interp) */ int -_Py_LegacyLocaleDetected(void) +_Py_LegacyLocaleDetected(const char *ctype_locale) { #ifndef MS_WINDOWS /* On non-Windows systems, the C locale is considered a legacy locale */ @@ -269,8 +270,7 @@ _Py_LegacyLocaleDetected(void) * the POSIX locale as a simple alias for the C locale, so * we may also want to check for that explicitly. */ - const char *ctype_loc = setlocale(LC_CTYPE, NULL); - return ctype_loc != NULL && strcmp(ctype_loc, "C") == 0; + return (ctype_locale != NULL && strcmp(ctype_locale, "C") == 0); #else /* Windows uses code pages instead of locales, so no locale is legacy */ return 0; @@ -286,7 +286,8 @@ static const char *_C_LOCALE_WARNING = static void _emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config) { - if (core_config->coerce_c_locale_warn && _Py_LegacyLocaleDetected()) { + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (core_config->preconfig.coerce_c_locale_warn && _Py_LegacyLocaleDetected(ctype_loc)) { PySys_FormatStderr("%s", _C_LOCALE_WARNING); } } @@ -479,16 +480,6 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p, } *interp_p = interp; - /* bpo-34008: For backward compatibility reasons, calling Py_Main() after - Py_Initialize() ignores the new configuration. */ - if (core_config->allocator != NULL) { - const char *allocator = _PyMem_GetAllocatorsName(); - if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) { - return _Py_INIT_USER_ERR("cannot modify memory allocator " - "after first Py_Initialize()"); - } - } - _PyCoreConfig_SetGlobalConfig(core_config); if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) { @@ -520,12 +511,6 @@ pycore_init_runtime(const _PyCoreConfig *core_config) return err; } - if (core_config->allocator != NULL) { - if (_PyMem_SetupAllocators(core_config->allocator) < 0) { - return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); - } - } - /* Py_Finalize leaves _Py_Finalizing set in order to help daemon * threads behave a little more gracefully at interpreter shutdown. * We clobber it here so the new interpreter can start with a clean @@ -750,25 +735,20 @@ _Py_InitializeCore(PyInterpreterState **interp_p, { assert(src_config != NULL); - PyMemAllocatorEx old_alloc; - _PyInitError err; + /* Set LC_CTYPE to the user preferred locale */ + _Py_SetLocaleFromEnv(LC_CTYPE); /* Copy the configuration, since _PyCoreConfig_Read() modifies it (and the input configuration is read only). */ _PyCoreConfig config = _PyCoreConfig_INIT; - /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); - - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_PyCoreConfig_Copy(&config, src_config) >= 0) { - err = _PyCoreConfig_Read(&config); - } - else { + _PyInitError err; + if (_PyCoreConfig_Copy(&config, src_config) < 0) { err = _Py_INIT_ERR("failed to copy core config"); + goto done; } - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + err = _PyCoreConfig_Read(&config, NULL); if (_Py_INIT_FAILED(err)) { goto done; } @@ -781,10 +761,7 @@ _Py_InitializeCore(PyInterpreterState **interp_p, } done: - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyCoreConfig_Clear(&config); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return err; } @@ -959,7 +936,7 @@ Py_InitializeEx(int install_sigs) _PyCoreConfig_Clear(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } } @@ -1431,7 +1408,7 @@ Py_NewInterpreter(void) PyThreadState *tstate; _PyInitError err = new_interpreter(&tstate); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } return tstate; @@ -1551,7 +1528,7 @@ initfsencoding(PyInterpreterState *interp) { _PyCoreConfig *config = &interp->core_config; - char *encoding = get_codec_name(config->filesystem_encoding); + char *encoding = get_codec_name(config->preconfig.filesystem_encoding); if (encoding == NULL) { /* Such error can only occurs in critical situations: no more memory, import a module of the standard library failed, etc. */ @@ -1562,13 +1539,13 @@ initfsencoding(PyInterpreterState *interp) /* Update the filesystem encoding to the normalized Python codec name. For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii" (Python codec name). */ - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; + PyMem_RawFree(config->preconfig.filesystem_encoding); + config->preconfig.filesystem_encoding = encoding; /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors global configuration variables. */ - if (_Py_SetFileSystemEncoding(config->filesystem_encoding, - config->filesystem_errors) < 0) { + if (_Py_SetFileSystemEncoding(config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors) < 0) { return _Py_INIT_NO_MEMORY(); } @@ -1770,13 +1747,13 @@ init_sys_streams(PyInterpreterState *interp) } #endif - char *codec_name = get_codec_name(config->stdio_encoding); + char *codec_name = get_codec_name(config->preconfig.stdio_encoding); if (codec_name == NULL) { return _Py_INIT_ERR("failed to get the Python codec name " "of the stdio encoding"); } - PyMem_RawFree(config->stdio_encoding); - config->stdio_encoding = codec_name; + PyMem_RawFree(config->preconfig.stdio_encoding); + config->preconfig.stdio_encoding = codec_name; /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1815,8 +1792,8 @@ init_sys_streams(PyInterpreterState *interp) * GUI apps don't have valid standard streams by default. */ std = create_stdio(config, iomod, fd, 0, "", - config->stdio_encoding, - config->stdio_errors); + config->preconfig.stdio_encoding, + config->preconfig.stdio_errors); if (std == NULL) goto error; PySys_SetObject("__stdin__", std); @@ -1826,8 +1803,8 @@ init_sys_streams(PyInterpreterState *interp) /* Set sys.stdout */ fd = fileno(stdout); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, - config->stdio_errors); + config->preconfig.stdio_encoding, + config->preconfig.stdio_errors); if (std == NULL) goto error; PySys_SetObject("__stdout__", std); @@ -1838,7 +1815,7 @@ init_sys_streams(PyInterpreterState *interp) /* Set sys.stderr, replaces the preliminary stderr */ fd = fileno(stderr); std = create_stdio(config, iomod, fd, 1, "", - config->stdio_encoding, + config->preconfig.stdio_encoding, "backslashreplace"); if (std == NULL) goto error; @@ -2072,12 +2049,17 @@ Py_FatalError(const char *msg) } void _Py_NO_RETURN -_Py_FatalInitError(_PyInitError err) +_Py_ExitInitError(_PyInitError err) { - /* On "user" error: exit with status 1. - For all other errors, call abort(). */ - int status = err.user_err ? 1 : -1; - fatal_error(err.prefix, err.msg, status); + if (err.exitcode >= 0) { + exit(err.exitcode); + } + else { + /* On "user" error: exit with status 1. + For all other errors, call abort(). */ + int status = err.user_err ? 1 : -1; + fatal_error(err.prefix, err.msg, status); + } } /* Clean up and exit */ diff --git a/Python/pystate.c b/Python/pystate.c index 6fe947d36466e51..a9a425c2f49cb91 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,6 +2,7 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" +#include "pycore_coreconfig.h" #include "pycore_pymem.h" #include "pycore_pystate.h" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index dd39305f39af6c7..d222b1546cd0754 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -421,7 +421,7 @@ sys_getfilesystemencoding_impl(PyObject *module) { PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; - return PyUnicode_FromString(config->filesystem_encoding); + return PyUnicode_FromString(config->preconfig.filesystem_encoding); } /*[clinic input] @@ -436,7 +436,7 @@ sys_getfilesystemencodeerrors_impl(PyObject *module) { PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); const _PyCoreConfig *config = &interp->core_config; - return PyUnicode_FromString(config->filesystem_errors); + return PyUnicode_FromString(config->preconfig.filesystem_errors); } /*[clinic input] @@ -2172,16 +2172,16 @@ make_flags(void) SetFlag(!config->write_bytecode); SetFlag(!config->user_site_directory); SetFlag(!config->site_import); - SetFlag(!config->use_environment); + SetFlag(!config->preconfig.use_environment); SetFlag(config->verbose); /* SetFlag(saw_unbuffered_flag); */ /* SetFlag(skipfirstline); */ SetFlag(config->bytes_warning); SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); - SetFlag(config->isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); - SetFlag(config->utf8_mode); + SetFlag(config->preconfig.isolated); + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode)); + SetFlag(config->preconfig.utf8_mode); #undef SetFlag if (PyErr_Occurred()) { From d94fd28400f145eb566eafe18b6f69465300457d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Mar 2019 02:15:57 +0100 Subject: [PATCH 2/6] Fix sys__enablelegacywindowsfsencoding_impl() --- Python/sysmodule.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d222b1546cd0754..27a0864f7c78e6b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1221,13 +1221,13 @@ sys__enablelegacywindowsfsencoding_impl(PyObject *module) return NULL; } - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; - PyMem_RawFree(config->filesystem_errors); - config->filesystem_errors = errors; + PyMem_RawFree(config->preconfig.filesystem_encoding); + config->preconfig.filesystem_encoding = encoding; + PyMem_RawFree(config->preconfig.filesystem_errors); + config->preconfig.filesystem_errors = errors; - if (_Py_SetFileSystemEncoding(config->filesystem_encoding, - config->filesystem_errors) < 0) { + if (_Py_SetFileSystemEncoding(config->preconfig.filesystem_encoding, + config->preconfig.filesystem_errors) < 0) { PyErr_NoMemory(); return NULL; } From 02a79a976b31c11749e18827216115d63b2549c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Mar 2019 02:23:19 +0100 Subject: [PATCH 3/6] Fix _PyPreConfig_WINDOWS_INIT --- Include/cpython/coreconfig.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index 72f943b2cf1b19a..59e54ca4a053f5d 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -140,16 +140,16 @@ typedef struct { } _PyPreConfig; #ifdef MS_WINDOWS -# define _PyCoreConfig_WINDOWS_INIT \ +# define _PyPreConfig_WINDOWS_INIT \ .legacy_windows_fs_encoding = -1, \ .legacy_windows_stdio = -1, #else -# define _PyCoreConfig_WINDOWS_INIT +# define _PyPreConfig_WINDOWS_INIT #endif #define _PyPreConfig_INIT \ (_PyPreConfig){ \ - _PyCoreConfig_WINDOWS_INIT \ + _PyPreConfig_WINDOWS_INIT \ .use_environment = -1, \ .isolated = -1, \ .utf8_mode = -1, \ @@ -360,7 +360,6 @@ typedef struct { .quiet = -1, \ .user_site_directory = -1, \ .buffered_stdio = -1, \ - _PyCoreConfig_WINDOWS_INIT \ ._install_importlib = 1, \ ._check_hash_pycs_mode = "default", \ ._frozen = -1} From a8fb926d78de097c1b9fca9f985110b2256d0755 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Mar 2019 02:24:16 +0100 Subject: [PATCH 4/6] Fix _freeze_importlib.c --- Programs/_freeze_importlib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 3f43757ecb1ff5e..e6c51a4a1dfbe82 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -77,9 +77,9 @@ main(int argc, char *argv[]) text[text_size] = '\0'; _PyCoreConfig config = _PyCoreConfig_INIT; + config.preconfig.use_environment = 0; config.user_site_directory = 0; config.site_import = 0; - config.use_environment = 0; config.program_name = L"./_freeze_importlib"; /* Don't install importlib, since it could execute outdated bytecode. */ config._install_importlib = 0; @@ -89,7 +89,7 @@ main(int argc, char *argv[]) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } sprintf(buf, "", name); From 326ea5c2736f19a168920d5b3e4e2bc8ccf0c9ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Mar 2019 02:43:43 +0100 Subject: [PATCH 5/6] Fix get_env_flag() Rename and move it to preconfig.c. --- Include/internal/pycore_coreconfig.h | 2 ++ Python/coreconfig.c | 38 ++++++++-------------------- Python/preconfig.c | 23 +++++++++++++++-- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index 3c04402175e6ba8..565f8e13bf41267 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -9,6 +9,7 @@ extern "C" { #endif /* helpers */ +PyAPI_FUNC(int) _Py_str_to_int(const char *str, int *result); PyAPI_FUNC(const wchar_t*) _Py_GetXOption(int nxoption, wchar_t **xoptions, wchar_t *name); PyAPI_FUNC(const wchar_t*) _Py_GetProgramFromArgv(int argc, wchar_t * const *argv); PyAPI_FUNC(_PyInitError) _PyArgv_Decode(const _PyArgv *args, int *argc_p, wchar_t*** argv_p); @@ -22,6 +23,7 @@ PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config); PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config); PyAPI_FUNC(_PyInitError) _PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args); PyAPI_FUNC(_PyInitError) _PyPreConfig_SetAllocator(_PyPreConfig *config); +PyAPI_FUNC(void) _PreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name); /* _PyCoreConfig */ PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig); diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 6d2369ca51a18e5..f9a6314cd73315e 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -688,8 +688,8 @@ config_init_hash_seed(_PyCoreConfig *config) } -static int -config_str_to_int(const char *str, int *result) +int +_Py_str_to_int(const char *str, int *result) { const char *endptr = str; errno = 0; @@ -724,49 +724,31 @@ config_wstr_to_int(const wchar_t *wstr, int *result) } -static void -get_env_flag(_PyPreConfig *config, int *flag, const char *name) -{ - const char *var = _PyPreConfig_GetEnv(config, name); - if (!var) { - return; - } - int value; - if (config_str_to_int(var, &value) < 0 || value < 0) { - /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ - value = 1; - } - if (*flag < value) { - *flag = value; - } -} - - static _PyInitError config_read_env_vars(_PyCoreConfig *config) { _PyPreConfig *preconfig = &config->preconfig; /* Get environment variables */ - get_env_flag(preconfig, &config->parser_debug, "PYTHONDEBUG"); - get_env_flag(preconfig, &config->verbose, "PYTHONVERBOSE"); - get_env_flag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); - get_env_flag(preconfig, &config->inspect, "PYTHONINSPECT"); + _PreConfig_GetEnvFlag(preconfig, &config->parser_debug, "PYTHONDEBUG"); + _PreConfig_GetEnvFlag(preconfig, &config->verbose, "PYTHONVERBOSE"); + _PreConfig_GetEnvFlag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); + _PreConfig_GetEnvFlag(preconfig, &config->inspect, "PYTHONINSPECT"); int dont_write_bytecode = 0; - get_env_flag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); + _PreConfig_GetEnvFlag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); if (dont_write_bytecode) { config->write_bytecode = 0; } int no_user_site_directory = 0; - get_env_flag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); + _PreConfig_GetEnvFlag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); if (no_user_site_directory) { config->user_site_directory = 0; } int unbuffered_stdio = 0; - get_env_flag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); + _PreConfig_GetEnvFlag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); if (unbuffered_stdio) { config->buffered_stdio = 0; } @@ -805,7 +787,7 @@ config_init_tracemalloc(_PyCoreConfig *config) const char *env = _PyCoreConfig_GetEnv(config, "PYTHONTRACEMALLOC"); if (env) { - if (!config_str_to_int(env, &nframe)) { + if (!_Py_str_to_int(env, &nframe)) { valid = (nframe >= 0); } else { diff --git a/Python/preconfig.c b/Python/preconfig.c index 39f6d5ff086875a..991bf42fcc81c8e 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -1,4 +1,5 @@ #include +#include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_getopt.h" #include "pycore_pylifecycle.h" @@ -707,6 +708,24 @@ preconfig_init_stdio_encoding(_PyPreConfig *config) } +void +_PreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name) +{ + const char *var = _PyPreConfig_GetEnv(config, name); + if (!var) { + return; + } + int value; + if (_Py_str_to_int(var, &value) < 0 || value < 0) { + /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ + value = 1; + } + if (*flag < value) { + *flag = value; + } +} + + static _PyInitError preconfig_read_env_vars(_PyPreConfig *config) { @@ -721,9 +740,9 @@ preconfig_read_env_vars(_PyPreConfig *config) } #ifdef MS_WINDOWS - get_env_flag(config, &config->legacy_windows_fs_encoding, + _PreConfig_GetEnvFlag(config, &config->legacy_windows_fs_encoding, "PYTHONLEGACYWINDOWSFSENCODING"); - get_env_flag(config, &config->legacy_windows_stdio, + _PreConfig_GetEnvFlag(config, &config->legacy_windows_stdio, "PYTHONLEGACYWINDOWSSTDIO"); #endif From 1eaeb4d5ae9cef53ca1e7e7ec80004a10cd54819 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Mar 2019 02:59:50 +0100 Subject: [PATCH 6/6] rename to _PyPreConfig_GetEnvFlag() --- Include/internal/pycore_coreconfig.h | 2 +- Python/coreconfig.c | 14 +++++++------- Python/preconfig.c | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index 565f8e13bf41267..db23995c4defacd 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -23,7 +23,7 @@ PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config); PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config); PyAPI_FUNC(_PyInitError) _PyPreConfig_InitFromArgv(_PyPreConfig *config, const _PyArgv *args); PyAPI_FUNC(_PyInitError) _PyPreConfig_SetAllocator(_PyPreConfig *config); -PyAPI_FUNC(void) _PreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name); +PyAPI_FUNC(void) _PyPreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name); /* _PyCoreConfig */ PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig); diff --git a/Python/coreconfig.c b/Python/coreconfig.c index f9a6314cd73315e..3a6afdb44a155fd 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -730,25 +730,25 @@ config_read_env_vars(_PyCoreConfig *config) _PyPreConfig *preconfig = &config->preconfig; /* Get environment variables */ - _PreConfig_GetEnvFlag(preconfig, &config->parser_debug, "PYTHONDEBUG"); - _PreConfig_GetEnvFlag(preconfig, &config->verbose, "PYTHONVERBOSE"); - _PreConfig_GetEnvFlag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); - _PreConfig_GetEnvFlag(preconfig, &config->inspect, "PYTHONINSPECT"); + _PyPreConfig_GetEnvFlag(preconfig, &config->parser_debug, "PYTHONDEBUG"); + _PyPreConfig_GetEnvFlag(preconfig, &config->verbose, "PYTHONVERBOSE"); + _PyPreConfig_GetEnvFlag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE"); + _PyPreConfig_GetEnvFlag(preconfig, &config->inspect, "PYTHONINSPECT"); int dont_write_bytecode = 0; - _PreConfig_GetEnvFlag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); + _PyPreConfig_GetEnvFlag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); if (dont_write_bytecode) { config->write_bytecode = 0; } int no_user_site_directory = 0; - _PreConfig_GetEnvFlag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); + _PyPreConfig_GetEnvFlag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE"); if (no_user_site_directory) { config->user_site_directory = 0; } int unbuffered_stdio = 0; - _PreConfig_GetEnvFlag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); + _PyPreConfig_GetEnvFlag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED"); if (unbuffered_stdio) { config->buffered_stdio = 0; } diff --git a/Python/preconfig.c b/Python/preconfig.c index 991bf42fcc81c8e..0d0a0d5044f743d 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -709,7 +709,7 @@ preconfig_init_stdio_encoding(_PyPreConfig *config) void -_PreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name) +_PyPreConfig_GetEnvFlag(_PyPreConfig *config, int *flag, const char *name) { const char *var = _PyPreConfig_GetEnv(config, name); if (!var) { @@ -740,9 +740,9 @@ preconfig_read_env_vars(_PyPreConfig *config) } #ifdef MS_WINDOWS - _PreConfig_GetEnvFlag(config, &config->legacy_windows_fs_encoding, + _PyPreConfig_GetEnvFlag(config, &config->legacy_windows_fs_encoding, "PYTHONLEGACYWINDOWSFSENCODING"); - _PreConfig_GetEnvFlag(config, &config->legacy_windows_stdio, + _PyPreConfig_GetEnvFlag(config, &config->legacy_windows_stdio, "PYTHONLEGACYWINDOWSSTDIO"); #endif