Skip to content

Commit

Permalink
Merge pull request #4243 from zooba/issue-4145
Browse files Browse the repository at this point in the history 
Fixes #4145 AD7 debugger can't attach to Python 3.7
  • Loading branch information
@zooba
zooba authored May 22, 2018
2 parents bce61de + 67faa75 commit d7ceb1b
Show file tree
Hide file tree
Showing 6 changed files with 127 additions and 44 deletions.
2 changes: 2 additions & 0 deletions Common/Tests/Utilities/VCCompiler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@ public class VCCompiler {
public static VCCompiler VC11_X86 { get { return FindVC("11.0", ProcessorArchitecture.X86); } }
public static VCCompiler VC12_X86 { get { return FindVC("12.0", ProcessorArchitecture.X86); } }
public static VCCompiler VC14_X86 { get { return FindVC("14.0", ProcessorArchitecture.X86); } }
public static VCCompiler VC15_X86 { get { return FindVC("15.0", ProcessorArchitecture.X86); } }
public static VCCompiler VC9_X64 { get { return FindVC("9.0", ProcessorArchitecture.Amd64); } }
public static VCCompiler VC10_X64 { get { return FindVC("10.0", ProcessorArchitecture.Amd64); } }
public static VCCompiler VC11_X64 { get { return FindVC("11.0", ProcessorArchitecture.Amd64); } }
public static VCCompiler VC12_X64 { get { return FindVC("12.0", ProcessorArchitecture.Amd64); } }
public static VCCompiler VC14_X64 { get { return FindVC("14.0", ProcessorArchitecture.Amd64); } }
public static VCCompiler VC15_X64 { get { return FindVC("15.0", ProcessorArchitecture.Amd64); } }

private VCCompiler(string bin, string bins, string include, string lib) {
BinPath = bin ?? string.Empty;
Expand Down
2 changes: 1 addition & 1 deletion Python/Product/Profiling/vspyprof.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def profile(file, globals_obj, locals_obj, profdll):
pyprofdll.InitProfiler.argtypes = [ctypes.c_void_p]
pyprofdll.InitProfiler.restype = ctypes.c_void_p

profiler = pyprofdll.CreateProfiler(sys.dllhandle)
profiler = pyprofdll.CreateProfiler(ctypes.c_void_p(sys.dllhandle))
if not profiler:
raise NotImplementedError("Profiling is currently not supported for " + sys.version)
handle = None
Expand Down
49 changes: 32 additions & 17 deletions Python/Product/PyDebugAttach/PyDebugAttach.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ typedef PyInterpreterState* (PyInterpreterState_Head)();
typedef PyThreadState* (PyInterpreterState_ThreadHead)(PyInterpreterState* interp);
typedef PyThreadState* (PyThreadState_Next)(PyThreadState *tstate);
typedef PyThreadState* (PyThreadState_Swap)(PyThreadState *tstate);
typedef PyThreadState* (_PyThreadState_UncheckedGet)();
typedef PyObject* (PyDict_New)();
typedef PyObject* (PyModule_New)(const char *name);
typedef PyObject* (PyModule_GetDict)(PyObject *module);
Expand Down Expand Up @@ -123,6 +124,7 @@ class InterpreterInfo {
InterpreterInfo(HMODULE module, bool debug) :
Interpreter(module),
CurrentThread(nullptr),
CurrentThreadGetter(nullptr),
NewThreadFunction(nullptr),
PyGILState_Ensure(nullptr),
Version(PythonVersion_Unknown),
Expand All @@ -141,6 +143,7 @@ class InterpreterInfo {

PyObjectHolder* NewThreadFunction;
PyThreadState** CurrentThread;
_PyThreadState_UncheckedGet *CurrentThreadGetter;

HMODULE Interpreter;
PyGILState_EnsureFunc* PyGILState_Ensure;
Expand Down Expand Up @@ -180,13 +183,16 @@ class InterpreterInfo {
}

bool EnsureCurrentThread() {
if (CurrentThread == nullptr) {
auto curPythonThread = (PyThreadState**)(void*)GetProcAddress(
Interpreter, "_PyThreadState_Current");
CurrentThread = curPythonThread;
if (CurrentThread == nullptr && CurrentThreadGetter == nullptr) {
CurrentThreadGetter = (_PyThreadState_UncheckedGet*)GetProcAddress(Interpreter, "_PyThreadState_UncheckedGet");
CurrentThread = (PyThreadState**)(void*)GetProcAddress(Interpreter, "_PyThreadState_Current");
}

return CurrentThread != nullptr;
return CurrentThread != nullptr || CurrentThreadGetter != nullptr;
}

PyThreadState *GetCurrentThread() {
return CurrentThreadGetter ? CurrentThreadGetter() : *CurrentThread;
}

private:
Expand Down Expand Up @@ -654,7 +660,7 @@ ConnectionInfo GetConnectionInfo() {
char* pBuf;

wchar_t fullMappingName[1024];
_snwprintf_s(fullMappingName, _countof(fullMappingName), L"PythonDebuggerMemory%d", GetCurrentProcessId());
_snwprintf_s(fullMappingName, sizeof(fullMappingName) / sizeof(fullMappingName[0]), L"PythonDebuggerMemory%d", GetCurrentProcessId());

hMapFile = OpenFileMapping(
FILE_MAP_ALL_ACCESS, // read/write access
Expand Down Expand Up @@ -807,7 +813,6 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {

// found initialized Python runtime, gather and check the APIs we need for a successful attach...
auto addPendingCall = (Py_AddPendingCall*)GetProcAddress(module, "Py_AddPendingCall");
auto curPythonThread = (PyThreadState**)(void*)GetProcAddress(module, "_PyThreadState_Current");
auto interpHead = (PyInterpreterState_Head*)GetProcAddress(module, "PyInterpreterState_Head");
auto gilEnsure = (PyGILState_Ensure*)GetProcAddress(module, "PyGILState_Ensure");
auto gilRelease = (PyGILState_Release*)GetProcAddress(module, "PyGILState_Release");
Expand Down Expand Up @@ -843,7 +848,6 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
strFromString = (PyString_FromString*)GetProcAddress(module, "PyString_FromString");
intFromSizeT = (PyInt_FromSize_t*)GetProcAddress(module, "PyInt_FromSize_t");
}
auto intervalCheck = (int*)GetProcAddress(module, "_Py_CheckInterval");
auto errOccurred = (PyErr_Occurred*)GetProcAddress(module, "PyErr_Occurred");
auto pyErrFetch = (PyErr_Fetch*)GetProcAddress(module, "PyErr_Fetch");
auto pyErrRestore = (PyErr_Restore*)GetProcAddress(module, "PyErr_Restore");
Expand All @@ -852,8 +856,7 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
auto pyGetAttr = (PyObject_GetAttrString*)GetProcAddress(module, "PyObject_GetAttrString");
auto pySetAttr = (PyObject_SetAttrString*)GetProcAddress(module, "PyObject_SetAttrString");
auto pyNone = (PyObject*)GetProcAddress(module, "_Py_NoneStruct");
auto getSwitchInterval = (_PyEval_GetSwitchInterval*)GetProcAddress(module, "_PyEval_GetSwitchInterval");
auto setSwitchInterval = (_PyEval_SetSwitchInterval*)GetProcAddress(module, "_PyEval_SetSwitchInterval");

auto boolFromLong = (PyBool_FromLong*)GetProcAddress(module, "PyBool_FromLong");
auto getThreadTls = (PyThread_get_key_value*)GetProcAddress(module, "PyThread_get_key_value");
auto setThreadTls = (PyThread_set_key_value*)GetProcAddress(module, "PyThread_set_key_value");
Expand All @@ -863,12 +866,22 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
auto pyUnicodeAsWideChar = (PyUnicode_AsWideChar*)GetProcAddress(module,
version < PythonVersion_33 ? "PyUnicodeUCS2_AsWideChar" : "PyUnicode_AsWideChar");

if (addPendingCall == nullptr || curPythonThread == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
// Either _PyThreadState_Current or _PyThreadState_UncheckedGet are required
auto curPythonThread = (PyThreadState**)(void*)GetProcAddress(module, "_PyThreadState_Current");
auto getPythonThread = (_PyThreadState_UncheckedGet*)GetProcAddress(module, "_PyThreadState_UncheckedGet");

// Either _Py_CheckInterval or _PyEval_[GS]etSwitchInterval are useful, but not required
auto intervalCheck = (int*)GetProcAddress(module, "_Py_CheckInterval");
auto getSwitchInterval = (_PyEval_GetSwitchInterval*)GetProcAddress(module, "_PyEval_GetSwitchInterval");
auto setSwitchInterval = (_PyEval_SetSwitchInterval*)GetProcAddress(module, "_PyEval_SetSwitchInterval");

if (addPendingCall == nullptr || interpHead == nullptr || gilEnsure == nullptr || gilRelease == nullptr || threadHead == nullptr ||
initThreads == nullptr || releaseLock == nullptr || threadsInited == nullptr || threadNext == nullptr || threadSwap == nullptr ||
pyDictNew == nullptr || pyCompileString == nullptr || pyEvalCode == nullptr || getDictItem == nullptr || call == nullptr ||
getBuiltins == nullptr || dictSetItem == nullptr || intFromLong == nullptr || pyErrRestore == nullptr || pyErrFetch == nullptr ||
errOccurred == nullptr || pyImportMod == nullptr || pyGetAttr == nullptr || pyNone == nullptr || pySetAttr == nullptr || boolFromLong == nullptr ||
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr || pyObjectRepr == nullptr || pyUnicodeAsWideChar == nullptr) {
getThreadTls == nullptr || setThreadTls == nullptr || delThreadTls == nullptr || pyObjectRepr == nullptr || pyUnicodeAsWideChar == nullptr ||
(curPythonThread == nullptr && getPythonThread == nullptr)) {
// we're missing some APIs, we cannot attach.
connInfo.ReportError(ConnError_PythonNotFound);
return false;
Expand Down Expand Up @@ -957,7 +970,9 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
SuspendThreads(suspendedThreads, addPendingCall, threadsInited);

if (!threadsInited()) {
if (*curPythonThread == nullptr) {
auto curPyThread = getPythonThread ? getPythonThread() : *curPythonThread;

if (curPyThread == nullptr) {
// no threads are currently running, it is safe to initialize multi threading.
PyGILState_STATE gilState;
if (version >= PythonVersion_34) {
Expand Down Expand Up @@ -1081,7 +1096,7 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {

auto repr = PyObjectHolder(isDebug, pyObjectRepr(value));
wchar_t reprText[0x1000] = {};
pyUnicodeAsWideChar(repr.ToPython(), reprText, _countof(reprText) - 1);
pyUnicodeAsWideChar(repr.ToPython(), reprText, sizeof(reprText) / sizeof(reprText[0]) - 1);
fputws(reprText, stderr);

connInfo.ReportErrorAfterAttachDone(ConnError_LoadDebuggerFailed);
Expand Down Expand Up @@ -1115,7 +1130,7 @@ bool DoAttach(HMODULE module, ConnectionInfo& connInfo, bool isDebug) {
unordered_set<PyThreadState*> seenThreads;
{
// find what index is holding onto the thread state...
auto curPyThread = *curPythonThread;
auto curPyThread = getPythonThread ? getPythonThread() : *curPythonThread;
int threadStateIndex = -1;
for (int i = 0; i < 100000; i++) {
void* value = getThreadTls(i);
Expand Down Expand Up @@ -1319,7 +1334,7 @@ int TraceGeneral(int interpreterId, PyObject *obj, PyFrameObject *frame, int wha

auto call = curInterpreter->GetCall();
if (call != nullptr && curInterpreter->EnsureCurrentThread()) {
auto curThread = *curInterpreter->CurrentThread;
auto curThread = curInterpreter->GetCurrentThread();

bool isDebug = new_thread->_isDebug;

Expand Down Expand Up @@ -1462,7 +1477,7 @@ PyGILState_STATE MyGilEnsureGeneral(DWORD interpreterId) {

if (res == PyGILState_UNLOCKED) {
if (curInterpreter->EnsureCurrentThread()) {
auto thread = *curInterpreter->CurrentThread;
auto thread = curInterpreter->GetCurrentThread();

if (thread != nullptr && curInterpreter->EnsureSetTrace()) {
SetInitialTraceFunc(interpreterId, thread);
Expand Down
13 changes: 10 additions & 3 deletions Python/Product/VsPyProf/PythonApi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ VsPyProf* VsPyProf::Create(HMODULE pythonModule) {
}

if ((major == 2 && (minor >= 4 && minor <= 7)) ||
(major == 3 && (minor >= 0 && minor <= 6))) {
(major == 3 && (minor >= 0 && minor <= 7))) {
return new VsPyProf(pythonModule,
major,
minor,
Expand Down Expand Up @@ -217,6 +217,9 @@ bool VsPyProf::GetUserToken(PyFrameObject* frameObj, DWORD_PTR& func, DWORD_PTR&
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject36*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject36*)codeObj)->co_firstlineno;
} else if (PyCodeObject37::IsFor(MajorVersion, MinorVersion)) {
RegisterName(func, ((PyCodeObject37*)codeObj)->co_name, &moduleName);
lineno = ((PyCodeObject37*)codeObj)->co_firstlineno;
}

// give the profiler the line number of this function
Expand Down Expand Up @@ -265,8 +268,10 @@ wstring VsPyProf::GetClassNameFromFrame(PyFrameObject* frameObj, PyObject *codeO
PyObject* self = nullptr;
if (PyFrameObject25_33::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject25_33*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_37::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_37*)frameObj)->f_localsplus[0];
} else if (PyFrameObject34_36::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject34_36*)frameObj)->f_localsplus[0];
} else if (PyFrameObject37::IsFor(MajorVersion, MinorVersion)) {
self = ((PyFrameObject37*)frameObj)->f_localsplus[0];
}
return GetClassNameFromSelf(self, codeObj);
}
Expand Down Expand Up @@ -296,6 +301,8 @@ wstring VsPyProf::GetClassNameFromSelf(PyObject* self, PyObject *codeObj) {
nameObj = ((PyCodeObject33_35*)codeObj)->co_name;
} else if (PyCodeObject36::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject36*)codeObj)->co_name;
} else if (PyCodeObject37::IsFor(MajorVersion, MinorVersion)) {
nameObj = ((PyCodeObject37*)codeObj)->co_name;
}
GetNameAscii(nameObj, codeName);

Expand Down
77 changes: 68 additions & 9 deletions Python/Product/VsPyProf/python.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,6 @@ class PyFrameObject : public PyVarObject {
to the current stack top. */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
};

#define CO_MAXBLOCKS 20
Expand All @@ -296,6 +295,7 @@ typedef struct {

class PyFrameObject25_33 : public PyFrameObject {
public:
PyObject * f_exc_type, *f_exc_value, *f_exc_traceback;
PyThreadState* f_tstate;
int f_lasti; /* Last instruction if called */
/* As of 2.3 f_lineno is only valid when tracing is active (i.e. when
Expand All @@ -311,8 +311,9 @@ class PyFrameObject25_33 : public PyFrameObject {
}
};

class PyFrameObject34_37 : public PyFrameObject {
class PyFrameObject34_36 : public PyFrameObject {
public:
PyObject * f_exc_type, *f_exc_value, *f_exc_traceback;
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;

Expand All @@ -326,7 +327,28 @@ class PyFrameObject34_37 : public PyFrameObject {
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 7;
return majorVersion == 3 && minorVersion >= 4 && minorVersion <= 6;
}
};

class PyFrameObject37 : public PyFrameObject {
public:
char f_trace_lines; /* Emit per-line trace events? */
char f_trace_opcodes; /* Emit per-opcode trace events? */
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;

int f_lasti; /* Last instruction if called */
/* As of 2.3 f_lineno is only valid when tracing is active (i.e. when
f_trace is set) -- at other times use PyCode_Addr2Line instead. */
int f_lineno; /* Current line number */
int f_iblock; /* index in f_blockstack */
char f_executing; /* whether the frame is still executing */
PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 7;
}
};

Expand Down Expand Up @@ -568,7 +590,7 @@ class PyThreadState_30_33 : public PyThreadState {
}
};

class PyThreadState_34_37_Base : public PyThreadState {
class PyThreadState_34_36 : public PyThreadState {
public:
PyThreadState *prev;
PyThreadState *next;
Expand Down Expand Up @@ -604,10 +626,7 @@ class PyThreadState_34_37_Base : public PyThreadState {
int gilstate_counter;

PyObject *async_exc; /* Asynchronous exception to raise */
};

class PyThreadState_34_36 : public PyThreadState_34_37_Base {
public:
long thread_id; /* Thread id where this tstate was created */
/* XXX signal handlers should also be here */

Expand All @@ -620,10 +639,50 @@ class PyThreadState_34_36 : public PyThreadState_34_37_Base {
}
};

class PyThreadState_37 : public PyThreadState_34_37_Base {
struct _PyErr_StackItem {
PyObject *exc_type, *exc_value, *exc_traceback;
struct _PyErr_StackItem *previous_item;
};

class PyThreadState_37 : public PyThreadState {
public:
PyThreadState * prev;
PyThreadState *next;
PyInterpreterState *interp;

PyFrameObject *frame;
int recursion_depth;
char overflowed; /* The stack has overflowed. Allow 50 more calls
to handle the runtime error. */
char recursion_critical; /* The current calls must not cause
a stack overflow. */
/* 'tracing' keeps track of the execution depth when tracing/profiling.
This is to prevent the actual trace/profile code from being recorded in
the trace/profile. */
int stackcheck_counter;

int tracing;
int use_tracing;

Py_tracefunc c_profilefunc;
Py_tracefunc c_tracefunc;
PyObject *c_profileobj;
PyObject *c_traceobj;

PyObject *curexc_type;
PyObject *curexc_value;
PyObject *curexc_traceback;

_PyErr_StackItem exc_state;
_PyErr_StackItem *exc_info;

PyObject *dict; /* Stores per-thread state */

int gilstate_counter;

PyObject *async_exc; /* Asynchronous exception to raise */

unsigned long thread_id; /* Thread id where this tstate was created */
/* XXX signal handlers should also be here */

static bool IsFor(int majorVersion, int minorVersion) {
return majorVersion == 3 && minorVersion >= 7;
Expand Down
Loading

0 comments on commit d7ceb1b

Please sign in to comment.