From cdc36c58544724a25f6348d97c92fd8531ad729e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 23 Jul 2021 22:28:07 +0100 Subject: [PATCH 1/6] fix leak in _PyCode_Update --- Objects/codeobject.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 278ee43860e112..79d0354e736c10 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -296,7 +296,7 @@ _PyCode_Validate(struct _PyCodeConstructor *con) } static void -init_code(PyCodeObject *co, struct _PyCodeConstructor *con) +init_code(PyCodeObject *co, struct _PyCodeConstructor *con, bool is_new) { if (con->localsplusnames) { int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); @@ -318,6 +318,11 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_nfreevars = 0; } + if(!is_new) { + Py_XDECREF(co->co_filename); + Py_XDECREF(co->co_name); + Py_XDECREF(co->co_qualname); + } Py_INCREF(con->filename); co->co_filename = con->filename; Py_INCREF(con->name); @@ -420,7 +425,8 @@ _PyCode_New(struct _PyCodeConstructor *con) PyErr_NoMemory(); return NULL; } - init_code(co, con); + + init_code(co, con, true); return co; } @@ -442,7 +448,7 @@ _PyCode_Update(struct _PyCodeConstructor *con, PyCodeObject *code) con->columntable = Py_None; } - init_code(code, con); // TODO: This leaks! + init_code(code, con, false); return code; } From 038c4dab3ccd21e99f920f83117f97cfdb9f2559 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 24 Jul 2021 19:57:42 +0100 Subject: [PATCH 2/6] init name fields in _PyCode_New --- Objects/codeobject.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 79d0354e736c10..3d098ef2f07835 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -296,7 +296,7 @@ _PyCode_Validate(struct _PyCodeConstructor *con) } static void -init_code(PyCodeObject *co, struct _PyCodeConstructor *con, bool is_new) +init_code(PyCodeObject *co, struct _PyCodeConstructor *con) { if (con->localsplusnames) { int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); @@ -318,17 +318,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con, bool is_new) co->co_nfreevars = 0; } - if(!is_new) { - Py_XDECREF(co->co_filename); - Py_XDECREF(co->co_name); - Py_XDECREF(co->co_qualname); - } Py_INCREF(con->filename); - co->co_filename = con->filename; + Py_XSETREF(co->co_filename, con->filename); Py_INCREF(con->name); - co->co_name = con->name; + Py_XSETREF(co->co_name, con->name); Py_INCREF(con->qualname); - co->co_qualname = con->qualname; + Py_XSETREF(co->co_qualname, con->qualname); + co->co_flags = con->flags; Py_XINCREF(con->code); @@ -425,8 +421,11 @@ _PyCode_New(struct _PyCodeConstructor *con) PyErr_NoMemory(); return NULL; } + co->co_filename = NULL; + co->co_name = NULL; + co->co_qualname = NULL; - init_code(co, con, true); + init_code(co, con); return co; } @@ -448,7 +447,7 @@ _PyCode_Update(struct _PyCodeConstructor *con, PyCodeObject *code) con->columntable = Py_None; } - init_code(code, con, false); + init_code(code, con); return code; } From daf97295f160ca1d311f9d608dd2ae76448cad0d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 24 Jul 2021 20:46:31 +0100 Subject: [PATCH 3/6] bump magic number (fixes crashes in test_importlib and test_scope) --- Lib/importlib/_bootstrap_external.py | 2 +- Programs/test_frozenmain.h | 49 ++++++++++++++-------------- Python/importlib_external.h | 2 +- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index ac178fabcd2027..71c61af774094c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -372,7 +372,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (9994).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (9996).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index f77a16cf79ac75..3a6782bfa1abc4 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,29 +1,20 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { - 99,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0, + 99,250,1,0,0,11,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,7,0,0,0,0,0,0,0,1,0,0, - 0,90,8,60,109,111,100,117,108,101,62,90,8,60,109,111, - 100,117,108,101,62,122,18,116,101,115,116,95,102,114,111,122, - 101,110,109,97,105,110,46,112,121,115,86,0,0,0,100,0, - 100,1,108,0,90,0,100,0,100,1,108,1,90,1,101,2, - 100,2,131,1,1,0,101,2,100,3,101,0,106,3,131,2, - 1,0,101,1,106,4,131,0,100,4,25,0,90,5,100,5, - 68,0,93,14,90,6,101,2,100,6,101,6,155,0,100,7, - 101,5,101,6,25,0,155,0,157,4,131,1,1,0,113,26, - 100,1,83,0,41,8,105,0,0,0,0,78,122,18,70,114, - 111,122,101,110,32,72,101,108,108,111,32,87,111,114,108,100, - 122,8,115,121,115,46,97,114,103,118,90,6,99,111,110,102, - 105,103,41,5,90,12,112,114,111,103,114,97,109,95,110,97, - 109,101,90,10,101,120,101,99,117,116,97,98,108,101,90,15, - 117,115,101,95,101,110,118,105,114,111,110,109,101,110,116,90, - 17,99,111,110,102,105,103,117,114,101,95,99,95,115,116,100, - 105,111,90,14,98,117,102,102,101,114,101,100,95,115,116,100, - 105,111,122,7,99,111,110,102,105,103,32,122,2,58,32,41, - 7,90,3,115,121,115,90,17,95,116,101,115,116,105,110,116, - 101,114,110,97,108,99,97,112,105,90,5,112,114,105,110,116, - 90,4,97,114,103,118,90,11,103,101,116,95,99,111,110,102, - 105,103,115,90,6,99,111,110,102,105,103,90,3,107,101,121, - 41,0,115,0,0,0,0,115,16,0,0,0,8,3,8,1, + 0,218,8,60,109,111,100,117,108,101,62,114,0,0,0,0, + 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,115,86,0,0,0,100,0,100,1,108,0,90, + 0,100,0,100,1,108,1,90,1,101,2,100,2,131,1,1, + 0,101,2,100,3,101,0,106,3,131,2,1,0,101,1,106, + 4,131,0,100,4,25,0,90,5,100,5,68,0,93,14,90, + 6,101,2,100,6,101,6,155,0,100,7,101,5,101,6,25, + 0,155,0,157,4,131,1,1,0,113,26,100,1,83,0,41, + 7,218,3,115,121,115,90,17,95,116,101,115,116,105,110,116, + 101,114,110,97,108,99,97,112,105,218,5,112,114,105,110,116, + 218,4,97,114,103,118,90,11,103,101,116,95,99,111,110,102, + 105,103,115,218,6,99,111,110,102,105,103,218,3,107,101,121, + 169,0,243,0,0,0,0,115,16,0,0,0,8,3,8,1, 8,2,12,1,12,1,8,1,26,7,4,249,115,18,0,0, 0,8,3,8,1,8,2,12,1,12,1,2,7,4,1,2, 249,30,7,115,86,0,0,0,1,11,1,11,1,11,1,11, @@ -31,6 +22,14 @@ unsigned char M_test_frozenmain[] = { 1,6,7,17,19,22,19,27,1,28,1,28,10,27,10,39, 10,41,42,50,10,51,1,7,12,2,1,42,1,42,5,8, 5,10,11,41,21,24,11,41,11,41,28,34,35,38,28,39, - 11,41,11,41,5,42,5,42,5,42,1,42,1,42,115,0, - 0,0,0, + 11,41,11,41,5,42,5,42,5,42,1,42,1,42,114,8, + 0,0,0,41,8,233,0,0,0,0,78,122,18,70,114,111, + 122,101,110,32,72,101,108,108,111,32,87,111,114,108,100,122, + 8,115,121,115,46,97,114,103,118,114,5,0,0,0,41,5, + 90,12,112,114,111,103,114,97,109,95,110,97,109,101,218,10, + 101,120,101,99,117,116,97,98,108,101,90,15,117,115,101,95, + 101,110,118,105,114,111,110,109,101,110,116,90,17,99,111,110, + 102,105,103,117,114,101,95,99,95,115,116,100,105,111,90,14, + 98,117,102,102,101,114,101,100,95,115,116,100,105,111,122,7, + 99,111,110,102,105,103,32,122,2,58,32, }; diff --git a/Python/importlib_external.h b/Python/importlib_external.h index d87b1dc1204127..501465be220d24 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -591,7 +591,7 @@ const unsigned char _Py_M__importlib_bootstrap_external[] = { 32,32,32,32,116,101,109,112,111,114,97,114,121,32,102,105, 108,101,32,105,115,32,97,116,116,101,109,112,116,101,100,46, 250,5,123,125,46,123,125,114,146,0,0,0,90,2,119,98, - 78,105,10,39,0,0,114,115,0,0,0,114,109,0,0,0, + 78,105,12,39,0,0,114,115,0,0,0,114,109,0,0,0, 115,2,0,0,0,13,10,90,11,95,95,112,121,99,97,99, 104,101,95,95,122,4,111,112,116,45,122,3,46,112,121,122, 4,46,112,121,119,122,4,46,112,121,99,41,1,218,12,111, From 0bc1f4ca068ddec61a6ae0d121fb19c344f95d61 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 25 Jul 2021 00:28:49 +0100 Subject: [PATCH 4/6] add question --- Python/marshal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/marshal.c b/Python/marshal.c index 052bc41bfd8526..492f90a6501c58 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -325,6 +325,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ + // TODO: is this enough? what if v is a list containing a code object? if (PyCode_Check(v)) return 0; /* never sharing code objects */ From f44cd7c660bd0e94cf990fbfe920673a547f8513 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 25 Jul 2021 16:33:31 +0100 Subject: [PATCH 5/6] Revert "add question" This reverts commit 0bc1f4ca068ddec61a6ae0d121fb19c344f95d61. --- Python/marshal.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/marshal.c b/Python/marshal.c index 492f90a6501c58..052bc41bfd8526 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -325,7 +325,6 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ - // TODO: is this enough? what if v is a list containing a code object? if (PyCode_Check(v)) return 0; /* never sharing code objects */ From 5bacfe73303cf6ce93f6d97fb3b44156ee78eedb Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 25 Jul 2021 16:56:24 +0100 Subject: [PATCH 6/6] move the co_*name decrefs to _PyCode_Update --- Objects/codeobject.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3d098ef2f07835..ad889ddb288ce2 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -319,11 +319,11 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) } Py_INCREF(con->filename); - Py_XSETREF(co->co_filename, con->filename); + co->co_filename = con->filename; Py_INCREF(con->name); - Py_XSETREF(co->co_name, con->name); + co->co_name = con->name; Py_INCREF(con->qualname); - Py_XSETREF(co->co_qualname, con->qualname); + co->co_qualname = con->qualname; co->co_flags = con->flags; @@ -447,6 +447,10 @@ _PyCode_Update(struct _PyCodeConstructor *con, PyCodeObject *code) con->columntable = Py_None; } + Py_XDECREF(code->co_filename); + Py_XDECREF(code->co_name); + Py_XDECREF(code->co_qualname); + init_code(code, con); return code;