diff --git a/Include/funcobject.h b/Include/funcobject.h index 77bb8c39aeb7500..ccd69456fe4c565 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -7,6 +7,48 @@ extern "C" { #endif +/* Function guard */ + +typedef struct { + PyObject ob_base; + + /* Initialize a guard: + + - Return 0 on success + - Return 1 if the guard will always fail: PyFunction_Specialize() must + ignore the specialization + - Raise an exception and return -1 on error */ + int (*init) (PyObject *guard, PyObject *func); + + /* Check a guard: + + - Return 0 on success + - Return 1 if the guard failed temporarely + - Return 2 if the guard will always fail: + calling the function must remove the specialized code + - Raise an exception and return -1 on error + + args is a C array for positional arguments followed by values of keyword + arguments. Keys of keyword arguments are stored as a tuple of strings in + kwnames. nargs is the number of positional parameters at the beginning + of stack. The size of kwnames gives the number of keyword values in the + stack after positional arguments. */ + int (*check) (PyObject *guard, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames); +} PyFuncGuardObject; + +PyAPI_DATA(PyTypeObject) PyFuncGuard_Type; + + +/* Specialized function */ + +typedef struct { + PyObject *code; /* callable or code object */ + Py_ssize_t nb_guard; + PyObject **guards; /* PyFuncGuardObject objects */ +} PySpecializedCode; + + /* Function objects and code objects should not be confused with each other: * * Function objects are created by the execution of the 'def' statement. @@ -33,6 +75,9 @@ typedef struct { PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_qualname; /* The qualified name */ + Py_ssize_t func_nb_specialized; + PySpecializedCode *func_specialized; + /* Invariant: * func_closure contains the bindings for func_code->co_freevars, so * PyTuple_Size(func_closure) == PyCode_GetNumFree(func_code) @@ -72,6 +117,45 @@ PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords( PyObject *kwnames); #endif +/* Specialize a function: add a specialized code with guards. code is a + * callable or code object, guards must be non-empty sequence of + * PyFuncGuardObject objects. Result: + * + * - Return 0 on success + * - Return 1 if the specialization has been ignored + * - Raise an exception and return -1 on error + * + * If code is a Python function, the code of the function is used as the + * specialized code. The specialized function must have the same parameter + * defaults, the same keyword parameter defaults, and it must not have + * specialized code. + * + * If code is a Python function or a code object, a new code object is created + * and the code name and first line number of the function code object are + * copied. The specialized code must have the same cell variables and the same + * free variables. + * */ +PyAPI_DATA(int) PyFunction_Specialize(PyObject *func, + PyObject *code, PyObject *guards); + +/* Get the list of specialized codes. + * + * Return a list of (code, guards) tuples where code is a callable or code + * object and guards is a list of PyFuncGuard objects. + * + * Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject*) PyFunction_GetSpecializedCodes(PyObject *func); + +/* Remove one specialized code of a function with its guards by its index. + * Return 0 on success or if the index does not exist. Raise an exception and + * return -1 on error. */ +PyAPI_FUNC(int) PyFunction_RemoveSpecialized(PyObject *func, Py_ssize_t index); + +/* Remove all specialized codes and guards from a function. + * Return 0 on success. Raise an exception and return -1 if func is not + * a function. */ +PyAPI_FUNC(int) PyFunction_RemoveAllSpecialized(PyObject *func); + /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ #define PyFunction_GET_CODE(func) \ diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index ca678b9deb7250f..7e259acb7931ef8 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -7,6 +7,7 @@ _testcapi = None import struct import collections +import types # The test cases here cover several paths through the function calling # code. They depend on the METH_XXX flag that is used to define a C diff --git a/Lib/test/test_func_specialize.py b/Lib/test/test_func_specialize.py new file mode 100644 index 000000000000000..e95f43b0542fc2b --- /dev/null +++ b/Lib/test/test_func_specialize.py @@ -0,0 +1,314 @@ +""" +Test the implementation of the PEP 510: Function specialization. +""" + +import unittest +from test import support + +# this test is written for CPython: it must be skipped if _testcapi is missing +_testcapi = support.import_module('_testcapi') + + +class MyGuard(_testcapi.PyGuard): + def __init__(self): + self.init_call = None + self.init_result = 0 + + self.check_call = None + self.check_result = 0 + + def check(self, *args, **kw): + self.check_call = (args, kw) + return self.check_result + + def init(self, func): + self.init_call = func + return self.init_result + + +class GuardError(Exception): + pass + + +class GuardCheck(unittest.TestCase): + def test_check_result(self): + guard = MyGuard() + self.assertEqual(guard(), 0) + + guard.check_result = 1 + self.assertEqual(guard(), 1) + + guard.check_result = 2 + self.assertEqual(guard(), 2) + + with self.assertRaises(RuntimeError): + guard.check_result = 3 + guard() + + def test_check_args(self): + guard = MyGuard() + guard(1, 2, 3, key1='value1', key2='value2') + + expected = ((1, 2, 3), {'key1': 'value1', 'key2': 'value2'}) + self.assertEqual(guard.check_call, expected) + + def test_check_error(self): + class GuardCheckError(MyGuard): + def check(self, *args, **kw): + raise GuardError + + guard = GuardCheckError() + with self.assertRaises(GuardError): + guard() + + def test_init_ok(self): + def func(): + pass + + def func2(): + pass + + guard = MyGuard() + _testcapi.func_specialize(func, func2.__code__, [guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + + self.assertIs(guard.init_call, func) + + def test_init_fail(self): + def func(): + pass + + def func2(): + pass + + guard = MyGuard() + guard.init_result = 1 + _testcapi.func_specialize(func, func2.__code__, [guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 0) + + self.assertIs(guard.init_call, func) + + def test_init_error(self): + def func(): + pass + + def func2(): + pass + + class GuardInitError(MyGuard): + def init(self, func): + raise GuardError + + guard = GuardInitError() + with self.assertRaises(GuardError): + _testcapi.func_specialize(func, func2.__code__, [guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 0) + + def test_init_bug(self): + def func(): + pass + + def func2(): + pass + + guard = MyGuard() + guard.init_result = 2 + with self.assertRaises(RuntimeError): + _testcapi.func_specialize(func, func2.__code__, [guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 0) + + +class SpecializeTests(unittest.TestCase): + def setUp(self): + self.guard = MyGuard() + + def test_get_specialize(self): + def func(): + return "slow" + + class FastFunc: + def __call__(self): + return "fast" + + fast_func = FastFunc() + _testcapi.func_specialize(func, fast_func, [self.guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + + specialized_code, guards = _testcapi.func_get_specialized(func)[0] + self.assertIs(specialized_code, fast_func) + self.assertEqual(len(guards), 1) + self.assertIs(guards[0], self.guard) + + def test_specialize_bytecode(self): + def func(): + return "slow" + + def fast_func(): + return "fast" + + self.assertEqual(func(), "slow") + + func_code = func.__code__ + fast_code = fast_func.__code__ + + # pass the fast_func code object + _testcapi.func_specialize(func, fast_code, [self.guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + + specialized_code, guards = _testcapi.func_get_specialized(func)[0] + self.assertEqual(specialized_code.co_code, fast_code.co_code) + + # specialize creates a new code object and copies the code name and the + # first number line + self.assertEqual(specialized_code.co_name, func_code.co_name) + self.assertEqual(specialized_code.co_firstlineno, func_code.co_firstlineno) + + self.assertEqual(func(), "fast") + + def test_specialize_builtin(self): + def func(obj): + return obj + + self.assertEqual(func("abc"), "abc") + + # pass builtin function len() + _testcapi.func_specialize(func, len, [self.guard]) + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + self.assertIs(_testcapi.func_get_specialized(func)[0][0], len) + + self.assertEqual(func("abc"), 3) + + def test_invalid_types(self): + def func(): + return "slow" + + def fast_func(): + return "fast" + + # specialize code must be a callable or code object + with self.assertRaises(TypeError): + _testcapi.func_specialize(func, 123, [self.guard]) + + # guards must be a sequence + with self.assertRaises(TypeError): + _testcapi.func_specialize(func, fast_func.__code__, 123) + + # guard must be a Guard + with self.assertRaises(TypeError): + _testcapi.func_specialize(func, fast_func.__code__, [123]) + + +class CallSpecializedTests(unittest.TestCase): + def setUp(self): + self.guard = MyGuard() + + def func(): + return "slow" + self.func = func + + def specialize(self): + def fast_func(): + return "fast" + + _testcapi.func_specialize(self.func, + fast_func.__code__, + [self.guard]) + + def test_guard_check_error(self): + self.specialize() + + self.guard.check_result = 3 + with self.assertRaises(RuntimeError): + self.func() + + def test_temporary_guard_fail(self): + self.specialize() + self.assertEqual(self.func(), "fast") + + # the guard temporarely fails + self.guard.check_result = 1 + self.assertEqual(self.func(), "slow") + + # the guard succeed again + self.guard.check_result = 0 + self.assertEqual(self.func(), "fast") + + def test_despecialize(self): + self.specialize() + self.assertEqual(len(_testcapi.func_get_specialized(self.func)), 1) + + # guard will always fail + self.guard.check_result = 2 + self.assertEqual(self.func(), "slow") + + # the call removed the specialized code + self.assertEqual(len(_testcapi.func_get_specialized(self.func)), 0) + + # check if the function still works + self.guard.check_result = 0 + self.assertEqual(self.func(), "slow") + + def test_despecialize_multiple_codes(self): + def func(): + return "slow" + + def fast_func1(): + return "fast1" + + guard1 = MyGuard() + guard1.check_result = 1 + _testcapi.func_specialize(func, fast_func1.__code__, [guard1]) + + def fast_func2(): + return "fast2" + + guard2 = MyGuard() + guard2.check_result = 1 + _testcapi.func_specialize(func, fast_func2.__code__, [guard2]) + + def fast_func3(): + return "fast3" + + guard3 = MyGuard() + _testcapi.func_specialize(func, fast_func3.__code__, [guard3]) + + self.assertEqual(func(), "fast3") + self.assertEqual(len(_testcapi.func_get_specialized(func)), 3) + + # guard 2 will always fail: + # fast_func2 specialized code will be removed + guard2.check_result = 2 + self.assertEqual(func(), "fast3") + self.assertEqual(len(_testcapi.func_get_specialized(func)), 2) + + # guard 3 will always fail: + # fast_func3 specialized code will be removed + guard3.check_result = 2 + self.assertEqual(func(), "slow") + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + + # guard 1 succeed again + guard1.check_result = 0 + self.assertEqual(func(), "fast1") + self.assertEqual(len(_testcapi.func_get_specialized(func)), 1) + + # guard 3 will always fail: + # the last specialized code (fast_func1) will be removed + guard1.check_result = 2 + self.assertEqual(func(), "slow") + self.assertEqual(len(_testcapi.func_get_specialized(func)), 0) + + def test_set_code(self): + self.specialize() + self.assertEqual(len(_testcapi.func_get_specialized(self.func)), 1) + + def func2(): + return "func2" + + # replacing the code object removes all specialized codes + self.func.__code__ = func2.__code__ + self.assertEqual(len(_testcapi.func_get_specialized(self.func)), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3844812ba288733..1d6cfca79a008d3 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1002,7 +1002,7 @@ class C(object): pass check(x, vsize('12P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P')) # function def func(): pass - check(func, size('12P')) + check(func, size('12PnP')) class c(): @staticmethod def foo(): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 83ba33dd7acfec0..d69d20650a6800c 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4149,6 +4149,74 @@ test_pyobject_fastcallkeywords(PyObject *self, PyObject *args) } +static PyObject * +func_specialize(PyObject *self, PyObject *args) +{ + PyObject *func, *code, *guards; + int res; + + if (!PyArg_ParseTuple(args, "O!OO:func_specialize", + &PyFunction_Type, &func, &code, &guards)) { + return NULL; + } + + res = PyFunction_Specialize(func, code, guards); + if (res < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +func_get_specialized(PyObject *self, PyObject *args) +{ + PyObject *func; + + if (!PyArg_ParseTuple(args, "O!:func_get_specialized", + &PyFunction_Type, &func)) { + return NULL; + } + + return PyFunction_GetSpecializedCodes(func); +} + +static PyObject * +func_remove_specialized(PyObject *self, PyObject *args) +{ + PyObject *func; + Py_ssize_t index; + + if (!PyArg_ParseTuple(args, "O!n:func_remove_specialized", + &PyFunction_Type, &func, &index)) { + return NULL; + } + + if (PyFunction_RemoveSpecialized(func, index) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +func_remove_all_specialized(PyObject *self, PyObject *args) +{ + PyObject *func; + + if (!PyArg_ParseTuple(args, "O!:func_remove_all_specialized", + &PyFunction_Type, &func)) { + return NULL; + } + + if (PyFunction_RemoveAllSpecialized(func) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4357,6 +4425,10 @@ static PyMethodDef TestMethods[] = { {"pyobject_fastcall", test_pyobject_fastcall, METH_VARARGS}, {"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS}, {"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords, METH_VARARGS}, + {"func_specialize", func_specialize, METH_VARARGS}, + {"func_get_specialized", func_get_specialized, METH_VARARGS}, + {"func_remove_specialized", func_remove_specialized, METH_VARARGS}, + {"func_remove_all_specialized", func_remove_all_specialized, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -4700,6 +4772,118 @@ static PyTypeObject awaitType = { }; +/* PyGuard */ + +static int +pyguard_init(PyObject *self, PyObject* func) +{ + PyObject *res_obj; + int res; + + res_obj = PyObject_CallMethod(self, "init", "O", func); + if (res_obj == NULL) { + return -1; + } + + res = PyLong_AsLong(res_obj); + Py_DECREF(res_obj); + if (res == -1 && PyErr_Occurred()) { + return -1; + } + + return res; +} + +static int +pyguard_check(PyObject *self, PyObject** stack, Py_ssize_t nargs, + PyObject *kwnames) +{ + PyObject *meth, *res_obj; + int res; + + meth = PyObject_GetAttrString(self, "check"); + if (meth == NULL) { + return -1; + } + + res_obj = _PyObject_FastCallKeywords(meth, stack, nargs, kwnames); + Py_DECREF(meth); + if (res_obj == NULL) { + return -1; + } + + res = PyLong_AsLong(res_obj); + Py_DECREF(res_obj); + if (res == -1 && PyErr_Occurred()) { + return -1; + } + + return res; +} + +static PyObject * +pyguard_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *op; + PyFuncGuardObject *self; + + op = PyFuncGuard_Type.tp_new(type, args, kwds); + if (op == NULL) + return NULL; + + self = (PyFuncGuardObject *)op; + self->init = pyguard_init; + self->check = pyguard_check; + + return op; +} + +static PyTypeObject PyGuard_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "PyGuard", + sizeof(PyFuncGuardObject), + 0, + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyFuncGuard_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + pyguard_new, /* tp_new */ + 0, /* tp_free */ +}; + + + + static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, "_testcapi", @@ -4741,6 +4925,14 @@ PyInit__testcapi(void) Py_INCREF(&awaitType); PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType); + /* tp_traverse is not "inherited" with PyType_Ready(), it should be + * inherited explicitly. */ + PyGuard_Type.tp_traverse = PyFuncGuard_Type.tp_traverse; + if (PyType_Ready(&PyGuard_Type) < 0) + return NULL; + Py_INCREF(&PyGuard_Type); + PyModule_AddObject(m, "PyGuard", (PyObject *)&PyGuard_Type); + PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); diff --git a/Objects/call.c b/Objects/call.c index 6c8a640cc59f21d..dcc8f13c83e77a1 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -294,24 +294,129 @@ function_code_fastcall(PyCodeObject *co, PyObject **args, Py_ssize_t nargs, } +static int +function_call_specialized(PyFunctionObject *func, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames, + PyObject **spe_code) +{ + assert(!PyErr_Occurred()); + /* for performance, this function must only be called if the called + function has specialized code */ + assert(func->func_nb_specialized > 0); + + for (Py_ssize_t i=0; i < func->func_nb_specialized; ) { + int res; + PySpecializedCode *spe; + Py_ssize_t j; + + spe = &func->func_specialized[i]; + + /* initialized to make the compiler happy, but res is always set + * in the loop */ + res = 0; + assert(spe->nb_guard >= 1); + for (j=0; j < spe->nb_guard; j++) { + PyFuncGuardObject *guard = (PyFuncGuardObject *)spe->guards[j]; + + res = guard->check((PyObject *)guard, stack, nargs, kwnames); + if (res) + break; + } + + if (!res) { + /* all guards succeded: use this specialized code */ + assert(!PyErr_Occurred()); + *spe_code = spe->code; + return 0; + } + + if (res == 2) { + /* a guard will always fail: remove the specialized code */ + if (PyFunction_RemoveSpecialized((PyObject *)func, i) < 0) { + return -1; + } + continue; + } + + if (res == -1) { + /* a guard raised an exception */ + assert(PyErr_Occurred()); + return -1; + } + + if (res != 1) { + PyErr_Format(PyExc_RuntimeError, + "guard check result must be in the range -1..2, " + "got %i", res); + return -1; + } + + /* a guard failed temporarely: try the next specialized code */ + i++; + } + assert(!PyErr_Occurred()); + + /* no specialized code was selected (all guards failed) */ + *spe_code = NULL; + return 0; +} + + PyObject * -_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, +_PyFunction_FastCallDict(PyObject *ofunc, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyFunctionObject *func; + PyCodeObject *co; + PyObject *globals; + PyObject *argdefs; PyObject *kwdefs, *closure, *name, *qualname; PyObject *kwtuple, **k; PyObject **d; Py_ssize_t nd, nk; PyObject *result; - assert(func != NULL); + assert(ofunc != NULL); assert(nargs >= 0); assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); + assert(PyFunction_Check(ofunc)); + func = (PyFunctionObject *)ofunc; + + co = (PyCodeObject *)PyFunction_GET_CODE(func); + globals = PyFunction_GET_GLOBALS(func); + argdefs = PyFunction_GET_DEFAULTS(func); + + if (func->func_nb_specialized) { + PyObject **stack; + PyObject *kwnames; + PyObject *spe_code; + int res; + + if (_PyStack_UnpackDict(args, nargs, kwargs, + &stack, &kwnames) < 0) { + return NULL; + } + + res = function_call_specialized(func, stack, nargs, kwnames, &spe_code); + if (stack != args) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + + if (res < 0) { + return NULL; + } + + if (spe_code) { + if (!PyCode_Check(spe_code)) { + return _PyObject_FastCallDict(spe_code, stack, nargs, kwargs); + } + co = (PyCodeObject *)spe_code; + } + } + if (co->co_kwonlyargcount == 0 && (kwargs == NULL || PyDict_GET_SIZE(kwargs) == 0) && co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) @@ -382,24 +487,50 @@ _PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, } PyObject * -_PyFunction_FastCallKeywords(PyObject *func, PyObject **stack, +_PyFunction_FastCallKeywords(PyObject *ofunc, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyFunctionObject *func; + PyCodeObject *co; + PyObject *globals; + PyObject *argdefs; PyObject *kwdefs, *closure, *name, *qualname; PyObject **d; - Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t nkwargs; Py_ssize_t nd; - assert(PyFunction_Check(func)); + assert(PyFunction_Check(ofunc)); + func = (PyFunctionObject *)ofunc; + assert(nargs >= 0); assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); + + nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); assert((nargs == 0 && nkwargs == 0) || stack != NULL); /* kwnames must only contains str strings, no subclass, and all keys must be unique */ + co = (PyCodeObject *)PyFunction_GET_CODE(func); + globals = PyFunction_GET_GLOBALS(func); + argdefs = PyFunction_GET_DEFAULTS(func); + + if (func->func_nb_specialized) { + PyObject *spe_code; + + if (function_call_specialized(func, stack, nargs, kwnames, + &spe_code) < 0) { + return NULL; + } + + if (spe_code) { + if (!PyCode_Check(spe_code)) { + return _PyObject_FastCallKeywords(spe_code, + stack, nargs, kwnames); + } + co = (PyCodeObject *)spe_code; + } + } + if (co->co_kwonlyargcount == 0 && nkwargs == 0 && co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 841000152f79a67..ff2f239de347068 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -5,6 +5,248 @@ #include "code.h" #include "structmember.h" +/* PyFuncGuard_Type */ + +static int +guard_check(PyObject *self, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + return 0; +} + +static int +guard_init(PyObject *self, PyObject *func) +{ + return 0; +} + +static PyObject* +guard_call(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyFuncGuardObject *guard = (PyFuncGuardObject *)self; + PyObject **args_stack, **stack; + Py_ssize_t nargs; + PyObject *kwnames; + int res; + + assert(PyTuple_CheckExact(args)); + args_stack = &PyTuple_GET_ITEM(args, 0); + nargs = PyTuple_GET_SIZE(args); + if (_PyStack_UnpackDict(args_stack, nargs, kwargs, + &stack, &kwnames) < 0) { + return NULL; + } + + res = guard->check(self, stack, nargs, kwnames); + if (stack != args_stack) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + + if (res < 0) { + return NULL; + } + + if (res > 2) { + PyErr_Format(PyExc_RuntimeError, + "guard check result must be in the range -1..2, " + "got %i", res); + return NULL; + } + + return PyLong_FromLong(res); +} + +static PyObject * +guard_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyFuncGuardObject *self; + + assert(type != NULL && type->tp_alloc != NULL); + self = (PyFuncGuardObject *)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + self->init = guard_init; + self->check = guard_check; + + return (PyObject *)self; +} + +static void +guard_dealloc(PyFuncGuardObject *op) +{ + PyObject_GC_UnTrack(op); + Py_TYPE(op)->tp_free(op); +} + +static int +guard_traverse(PyObject *self, visitproc visit, void *arg) +{ + return 0; +} + +PyTypeObject PyFuncGuard_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "FuncGuard", + sizeof(PyFuncGuardObject), + 0, + (destructor)guard_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + guard_call, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + guard_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + guard_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + +/* PySpecializedCode */ + +static void +specode_guards_dealloc(Py_ssize_t nguard, PyObject **guards) +{ + Py_ssize_t i; + for (i=0; i < nguard; i++) + Py_DECREF(guards[i]); + PyMem_Free(guards); +} + +static void +specode_dealloc(PySpecializedCode *f) +{ + Py_CLEAR(f->code); + specode_guards_dealloc(f->nb_guard, f->guards); +} + +static PyObject* +specode_as_tuple(PySpecializedCode* spe) +{ + PyObject *guards = NULL, *tuple; + Py_ssize_t i; + + guards = PyList_New(spe->nb_guard); + if (guards == NULL) + return NULL; + + for (i=0; inb_guard; i++) { + PyObject *guard = spe->guards[i]; + + Py_INCREF(guard); + PyList_SET_ITEM(guards, i, guard); + } + + tuple = PyTuple_New(2); + if (tuple == NULL) { + Py_XDECREF(guards); + return NULL; + } + + Py_INCREF(spe->code); + PyTuple_SET_ITEM(tuple, 0, spe->code); + PyTuple_SET_ITEM(tuple, 1, guards); + return tuple; +} + + +/* PyFunctionObject */ + +static int +func_remove_specialized(PyFunctionObject *op, Py_ssize_t index) +{ + Py_ssize_t size; + PySpecializedCode *specialized; + + if (index >= op->func_nb_specialized) { + /* invalid index: do nothing */ + return 0; + } + + specode_dealloc(&op->func_specialized[index]); + + op->func_nb_specialized--; + if (index < op->func_nb_specialized) { + size = (op->func_nb_specialized - index) * sizeof(op->func_specialized[0]); + memmove(&op->func_specialized[index], &op->func_specialized[index+1], size); + } + + size = op->func_nb_specialized * sizeof(PySpecializedCode); + specialized = PyMem_Realloc(op->func_specialized, size); + if (specialized != NULL) { + op->func_specialized = specialized; + } + else { + /* shrinking a memory block is not supposed the fail, but it's + * doesn't matter if the array is a little bit larger if realloc + * failed. */ + } + return 0; +} + +int +PyFunction_RemoveSpecialized(PyObject *func, Py_ssize_t index) +{ + if (!PyFunction_Check(func)) { + PyErr_BadInternalCall(); + return -1; + } + if (index < 0) { + PyErr_SetString(PyExc_ValueError, "index must be >= 0"); + return -1; + } + func_remove_specialized((PyFunctionObject *)func, index); + return 0; +} + +static void +func_remove_all_specialized(PyFunctionObject *func) +{ + Py_ssize_t i; + + for (i=0; i < func->func_nb_specialized; i++) + specode_dealloc(&func->func_specialized[i]); + func->func_nb_specialized = 0; +} + +int +PyFunction_RemoveAllSpecialized(PyObject *func) +{ + if (!PyFunction_Check(func)) { + PyErr_BadInternalCall(); + return -1; + } + func_remove_all_specialized((PyFunctionObject *)func); + return 0; +} + PyObject * PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) { @@ -61,6 +303,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_qualname = op->func_name; Py_INCREF(op->func_qualname); + op->func_nb_specialized = 0; + op->func_specialized = NULL; + _PyObject_GC_TRACK(op); return (PyObject *)op; } @@ -271,6 +516,7 @@ func_set_code(PyFunctionObject *op, PyObject *value) } Py_INCREF(value); Py_XSETREF(op->func_code, value); + func_remove_all_specialized(op); return 0; } @@ -538,6 +784,8 @@ func_dealloc(PyFunctionObject *op) Py_XDECREF(op->func_closure); Py_XDECREF(op->func_annotations); Py_XDECREF(op->func_qualname); + func_remove_all_specialized(op); + PyMem_Free(op->func_specialized); PyObject_GC_Del(op); } @@ -551,6 +799,8 @@ func_repr(PyFunctionObject *op) static int func_traverse(PyFunctionObject *f, visitproc visit, void *arg) { + Py_ssize_t i, j; + Py_VISIT(f->func_code); Py_VISIT(f->func_globals); Py_VISIT(f->func_module); @@ -562,6 +812,14 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); Py_VISIT(f->func_qualname); + + for (i=0; i < f->func_nb_specialized; i++) { + PySpecializedCode *spe = &f->func_specialized[i]; + + Py_VISIT(spe->code); + for (j=0; j < spe->nb_guard; j++) + Py_VISIT(spe->guards[j]); + } return 0; } @@ -587,6 +845,339 @@ func_descr_get(PyObject *func, PyObject *obj, PyObject *type) return PyMethod_New(func, obj); } + +static int +func_guards_from_list(PyObject *self, PyObject *guard_list, + Py_ssize_t *nb_guard, PyObject ***pguards) +{ + PyObject *seq, *item; + Py_ssize_t size, n, i; + PyObject **guards = NULL; + int res = 1; + + *nb_guard = 0; + *pguards = NULL; + + seq = PySequence_Fast(guard_list, "guards must be a iterable"); + if (seq == NULL) + return -1; + + n = PySequence_Fast_GET_SIZE(seq); + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "need at least one guard"); + goto error; + } + + size = n * sizeof(guards[0]); + guards = PyMem_Malloc(size); + if (guards == NULL) { + PyErr_NoMemory(); + goto error; + } + + for (i=0; itp_name); + goto error; + } + + guard = (PyFuncGuardObject *)item; + + init_res = guard->init((PyObject *)guard, (PyObject *)self); + if (init_res == -1) + goto error; + if (init_res == 1) + goto cleanup; + if (init_res) { + PyErr_Format(PyExc_RuntimeError, + "guard init result must be in the range -1..1, " + "got %i", res); + goto error; + } + + Py_INCREF(guard); + guards[i] = (PyObject *)guard; + (*nb_guard)++; + } + + Py_DECREF(seq); + *pguards = guards; + return 0; + +error: + res = -1; +cleanup: + specode_guards_dealloc((*nb_guard), guards); + *nb_guard = 0; + Py_DECREF(seq); + return res; +} + +static PyCodeObject* +func_specialize_code(PyFunctionObject *self, PyObject *obj) +{ + PyFunctionObject *func = NULL, *self_func; + PyCodeObject *self_code, *code; + int equals; + + self_func = (PyFunctionObject *)self; + self_code = (PyCodeObject*)self->func_code; + + if (PyFunction_Check(obj)) { + func = (PyFunctionObject *)obj; + code = (PyCodeObject*)func->func_code; + } + else { + assert(PyCode_Check(obj)); + code = (PyCodeObject *)obj; + } + + /* check cell variables */ + if (code->co_cellvars != self_code->co_cellvars + && (code->co_cellvars == NULL || self_code->co_cellvars == NULL)) { + equals = 0; + } + else { + equals = PyObject_RichCompareBool(code->co_cellvars, + self_code->co_cellvars, + Py_EQ); + } + if (equals == -1) { + return NULL; + } + if (!equals) { + PyErr_SetString(PyExc_ValueError, + "specialized bytecode uses " + "different cell variables"); + return NULL; + } + + /* check free variables */ + if (code->co_freevars != self_code->co_freevars + && (code->co_freevars == NULL || self_code->co_freevars == NULL)) { + equals = 0; + } + else { + equals = PyObject_RichCompareBool(code->co_freevars, + self_code->co_freevars, + Py_EQ); + } + if (equals == -1) { + return NULL; + } + if (!equals) { + PyErr_SetString(PyExc_ValueError, + "specialized bytecode uses " + "different free variables"); + return NULL; + } + + /* check code */ + if (code->co_argcount != self_code->co_argcount + || code->co_kwonlyargcount != self_code->co_kwonlyargcount) + { + PyErr_SetString(PyExc_ValueError, + "specialized bytecode doesn't have " + "the same number of parameters"); + return NULL; + } + + if (func != NULL) { + if ((PyObject *)func == (PyObject *)self) { + PyErr_SetString(PyExc_ValueError, + "a function cannot specialize itself"); + return NULL; + } + + if (func->func_nb_specialized) { + PyErr_SetString(PyExc_ValueError, + "cannot specialize a function with another " + "function which is already specialized"); + return NULL; + } + + /* check func defaults */ + if (func->func_defaults != self_func->func_defaults + && (func->func_defaults == NULL + || self_func->func_defaults == NULL)) { + equals = 0; + } + else { + equals = PyObject_RichCompareBool(func->func_defaults, + self_func->func_defaults, + Py_EQ); + } + if (equals == -1) { + return NULL; + } + if (!equals) { + PyErr_SetString(PyExc_ValueError, + "specialized function doesn't have " + "the same parameter defaults"); + return NULL; + } + + /* check func kwdefaults */ + if (func->func_kwdefaults != self_func->func_kwdefaults + && (func->func_kwdefaults == NULL + || self_func->func_kwdefaults == NULL)) { + equals = 0; + } + else { + equals = PyObject_RichCompareBool(func->func_kwdefaults, + self_func->func_kwdefaults, + Py_EQ); + } + if (equals == -1) { + return NULL; + } + if (!equals) { + PyErr_SetString(PyExc_ValueError, + "specialized function doesn't have " + "the same keyword parameter defaults"); + return NULL; + } + } + + /* create a new code object to replace the code name and the first line + * number */ + return PyCode_New(code->co_argcount, + code->co_kwonlyargcount, + code->co_nlocals, + code->co_stacksize, + code->co_flags, + code->co_code, + code->co_consts, + code->co_names, + code->co_varnames, + code->co_freevars, + code->co_cellvars, + code->co_filename, + self_code->co_name, /* copy name */ + self_code->co_firstlineno, /* copy first line number */ + code->co_lnotab); +} + +int +PyFunction_Specialize(PyObject *func, PyObject *obj, PyObject *guard_list) +{ + PyFunctionObject *self; + int res = -1; + Py_ssize_t nb_guard = 0; + PyObject **guards = NULL; + int get_res; + Py_ssize_t size; + PyObject *new_code = NULL, *code; + PySpecializedCode *specialized, *spefunc; + + if (!PyFunction_Check(func)) { + PyErr_BadInternalCall(); + goto exit; + } + self = (PyFunctionObject *)func; + + nb_guard = 0; + guards = NULL; + + if (PyFunction_Check(obj) || PyCode_Check(obj)) { + new_code = (PyObject *)func_specialize_code(self, obj); + if (new_code == NULL) + goto exit; + code = new_code; + } + else if (PyCallable_Check(obj)) { + code = obj; + } + else { + PyErr_Format(PyExc_TypeError, "function or code expected, got %s", + Py_TYPE(obj)->tp_name); + goto exit; + } + + /* get guards */ + get_res = func_guards_from_list((PyObject *)self, guard_list, + &nb_guard, &guards); + if (get_res < 0) { + goto exit; + } + if (get_res) { + /* a guard failed: ignore the specialization */ + res = 0; + goto exit; + } + + /* add specialized */ + size = sizeof(PySpecializedCode); + if (self->func_nb_specialized > PY_SSIZE_T_MAX / size - 1) { + PyErr_NoMemory(); + goto exit; + } + size = (self->func_nb_specialized + 1) * size; + + specialized = PyMem_Realloc(self->func_specialized, size); + if (specialized == NULL) { + PyErr_NoMemory(); + goto exit; + } + self->func_specialized = specialized; + spefunc = &specialized[self->func_nb_specialized]; + + /* set specialized attributes */ + Py_INCREF(code); + spefunc->code = code; + Py_XDECREF(new_code); + spefunc->nb_guard = nb_guard; + spefunc->guards = guards; + + self->func_nb_specialized++; + return 1; + +exit: + Py_XDECREF(new_code); + specode_guards_dealloc(nb_guard, guards); + return res; +} + + +PyObject * +PyFunction_GetSpecializedCodes(PyObject *func) +{ + PyFunctionObject *op; + PyObject *list = NULL; + Py_ssize_t i; + + if (!PyFunction_Check(func)) { + PyErr_BadInternalCall(); + return NULL; + } + op = (PyFunctionObject *)func; + + list = PyList_New(op->func_nb_specialized); + if (list == NULL) { + return NULL; + } + + for (i=0; i < op->func_nb_specialized; i++) { + PyObject *item; + + item = specode_as_tuple(&op->func_specialized[i]); + if (item == NULL) { + Py_DECREF(list); + return NULL; + } + + PyList_SET_ITEM(list, i, item); + } + return list; +} + + PyTypeObject PyFunction_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "function", diff --git a/Objects/object.c b/Objects/object.c index 2ba6e572ea61ff9..c1c1f53443eabb1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1758,6 +1758,9 @@ _Py_ReadyTypes(void) if (PyType_Ready(&PyFrame_Type) < 0) Py_FatalError("Can't initialize frame type"); + if (PyType_Ready(&PyFuncGuard_Type) < 0) + Py_FatalError("Can't initialize function guard"); + if (PyType_Ready(&PyCFunction_Type) < 0) Py_FatalError("Can't initialize builtin function type");