Stepping in multi-threaded case should continue other threads. Fixes #1372

src/ptvsd/_vendored/pydevd/.travis.yml

@@ -1,5 +1,8 @@
 language: python
 
+services:
+  - xvfb
+
 matrix:
   include:
     # Note: python is always 2.7 because it's the installed version
@@ -85,12 +88,8 @@ before_install:
   # Jython setup
   - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then wget $JYTHON_URL -O jython_installer.jar; java -jar jython_installer.jar -s -d $HOME/jython; export PATH=$HOME/jython:$HOME/jython/bin:$PATH; fi
   - if [ "$PYDEVD_TEST_JYTHON" == "YES" ]; then jython -c "print('')"; fi
-  # The next couple lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda
-  - sudo rm -rf /dev/shm
-  - sudo ln -s /run/shm /dev/shm
   # Fix issue with testGui
   - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
 # Install packages
 install:
   # Both

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_api.py

@@ -4,15 +4,16 @@
 from _pydevd_bundle import pydevd_utils
 from _pydevd_bundle.pydevd_additional_thread_info import set_additional_thread_info
 from _pydevd_bundle.pydevd_comm import (InternalGetThreadStack, internal_get_completions,
-    pydevd_find_thread_by_id, InternalStepThread, InternalSetNextStatementThread, internal_reload_code,
+    pydevd_find_thread_by_id, InternalSetNextStatementThread, internal_reload_code,
     InternalGetVariable, InternalGetArray, InternalLoadFullValue,
     internal_get_description, internal_get_frame, internal_evaluate_expression, InternalConsoleExec,
     internal_get_variable_json, internal_change_variable, internal_change_variable_json,
-    internal_evaluate_expression_json, internal_set_expression_json, internal_get_exception_details_json)
+    internal_evaluate_expression_json, internal_set_expression_json, internal_get_exception_details_json,
+    internal_step_in_thread, internal_run_thread)
 from _pydevd_bundle.pydevd_comm_constants import (CMD_THREAD_SUSPEND, file_system_encoding,
     CMD_STEP_INTO_MY_CODE, CMD_STOP_ON_START)
 from _pydevd_bundle.pydevd_constants import (get_current_thread_id, set_protocol, get_protocol,
-    HTTP_JSON_PROTOCOL, JSON_PROTOCOL, STATE_RUN, IS_PY3K, DebugInfoHolder, dict_keys)
+    HTTP_JSON_PROTOCOL, JSON_PROTOCOL, IS_PY3K, DebugInfoHolder, dict_keys)
 from _pydevd_bundle.pydevd_net_command_factory_json import NetCommandFactoryJson
 from _pydevd_bundle.pydevd_net_command_factory_xml import NetCommandFactory
 import pydevd_file_utils
@@ -140,11 +141,8 @@ def request_resume_thread(self, thread_id):
         for t in threads:
             if t is None:
                 continue
-            additional_info = set_additional_thread_info(t)
-            additional_info.pydev_original_step_cmd = -1
-            additional_info.pydev_step_cmd = -1
-            additional_info.pydev_step_stop = None
-            additional_info.pydev_state = STATE_RUN
+
+            internal_run_thread(t, set_additional_thread_info=set_additional_thread_info)
 
     def request_completions(self, py_db, seq, thread_id, frame_id, act_tok, line=-1, column=-1):
         py_db.post_method_as_internal_command(
@@ -167,14 +165,19 @@ def request_exception_info_json(self, py_db, request, thread_id, max_frames):
             thread_id,
             max_frames,
             set_additional_thread_info=set_additional_thread_info,
-            iter_visible_frames_info=py_db.cmd_factory._iter_visible_frames_info)
+            iter_visible_frames_info=py_db.cmd_factory._iter_visible_frames_info,
+        )
 
     def request_step(self, py_db, thread_id, step_cmd_id):
         t = pydevd_find_thread_by_id(thread_id)
         if t:
-            int_cmd = InternalStepThread(thread_id, step_cmd_id)
-            py_db.post_internal_command(int_cmd, thread_id)
-
+            py_db.post_method_as_internal_command(
+                thread_id,
+                internal_step_in_thread,
+                thread_id,
+                step_cmd_id,
+                set_additional_thread_info=set_additional_thread_info,
+            )
         elif thread_id.startswith('__frame__:'):
             sys.stderr.write("Can't make tasklet step command: %s\n" % (thread_id,))
 
@@ -562,6 +565,9 @@ def set_project_roots(self, py_db, project_roots):
         '''
         py_db.set_project_roots(project_roots)
 
+    def set_stepping_resumes_all_threads(self, py_db, stepping_resumes_all_threads):
+        py_db.stepping_resumes_all_threads = stepping_resumes_all_threads
+
     # Add it to the namespace so that it's available as PyDevdAPI.ExcludeFilter
     from _pydevd_bundle.pydevd_filtering import ExcludeFilter  # noqa
 

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py

@@ -89,7 +89,7 @@
     from urllib.parse import quote_plus, unquote_plus  # @Reimport @UnresolvedImport
 
 import pydevconsole
-from _pydevd_bundle import pydevd_vars
+from _pydevd_bundle import pydevd_vars, pydevd_utils
 import pydevd_tracing
 from _pydevd_bundle import pydevd_xml
 from _pydevd_bundle import pydevd_vm_type
@@ -607,29 +607,27 @@ def do_it(self, dbg):
             self._cmd = None
 
 
-class InternalRunThread(InternalThreadCommand):
+def internal_run_thread(thread, set_additional_thread_info):
+    info = set_additional_thread_info(thread)
+    info.pydev_original_step_cmd = -1
+    info.pydev_step_cmd = -1
+    info.pydev_step_stop = None
+    info.pydev_state = STATE_RUN
 
-    def do_it(self, dbg):
-        t = pydevd_find_thread_by_id(self.thread_id)
-        if t:
-            t.additional_info.pydev_original_step_cmd = -1
-            t.additional_info.pydev_step_cmd = -1
-            t.additional_info.pydev_step_stop = None
-            t.additional_info.pydev_state = STATE_RUN
 
+def internal_step_in_thread(py_db, thread_id, cmd_id, set_additional_thread_info):
+    thread_to_step = pydevd_find_thread_by_id(thread_id)
+    if thread_to_step:
+        info = set_additional_thread_info(thread_to_step)
+        info.pydev_original_step_cmd = cmd_id
+        info.pydev_step_cmd = cmd_id
+        info.pydev_state = STATE_RUN
 
-class InternalStepThread(InternalThreadCommand):
-
-    def __init__(self, thread_id, cmd_id):
-        self.thread_id = thread_id
-        self.cmd_id = cmd_id
-
-    def do_it(self, dbg):
-        t = pydevd_find_thread_by_id(self.thread_id)
-        if t:
-            t.additional_info.pydev_original_step_cmd = self.cmd_id
-            t.additional_info.pydev_step_cmd = self.cmd_id
-            t.additional_info.pydev_state = STATE_RUN
+    if py_db.stepping_resumes_all_threads:
+        threads = pydevd_utils.get_non_pydevd_threads()
+        for t in threads:
+            if t is not thread_to_step:
+                internal_run_thread(t, set_additional_thread_info)
 
 
 class InternalSetNextStatementThread(InternalThreadCommand):

src/ptvsd/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py

@@ -259,6 +259,9 @@ def _resolve_remote_root(self, local_root, remote_root):
 
     def _set_debug_options(self, py_db, args, start_reason):
         rules = args.get('rules')
+        stepping_resumes_all_threads = args.get('steppingResumesAllThreads', True)
+        self.api.set_stepping_resumes_all_threads(py_db, stepping_resumes_all_threads)
+
         exclude_filters = []
 
         if rules is not None:

src/ptvsd/_vendored/pydevd/pydevd.py

@@ -478,6 +478,9 @@ def __init__(self, set_as_global=True):
         # If True, pydevd will send a single notification when all threads are suspended/resumed.
         self._threads_suspended_single_notification = ThreadsSuspendedSingleNotification(self)
 
+        # If True a step command will do a step in one thread and will also resume all other threads.
+        self.stepping_resumes_all_threads = False
+
         self._local_thread_trace_func = threading.local()
 
         # Bind many locals to the debugger because upon teardown those names may become None
@@ -1069,11 +1072,12 @@ def return_control():
 
     def _activate_mpl_if_needed(self):
         if len(self.mpl_modules_for_patching) > 0:
-            if is_current_thread_main_thread():
+            if is_current_thread_main_thread():  # Note that we call only in the main thread.
                 for module in dict_keys(self.mpl_modules_for_patching):
                     if module in sys.modules:
-                        activate_function = self.mpl_modules_for_patching.pop(module)
-                        activate_function()
+                        activate_function = self.mpl_modules_for_patching.pop(module, None)
+                        if activate_function is not None:
+                            activate_function()
                         self.mpl_in_use = True
 
     def _call_mpl_hook(self):
@@ -1492,19 +1496,25 @@ def _do_wait_suspend(self, thread, frame, event, arg, suspend_type, from_this_th
         info = thread.additional_info
         keep_suspended = False
 
-        if info.pydev_state == STATE_SUSPEND and not self._finish_debugging_session:
-            in_main_thread = is_current_thread_main_thread()
+        with self._main_lock:  # Use lock to check if suspended state changed
+            activate_matplotlib = info.pydev_state == STATE_SUSPEND and not self._finish_debugging_session
+
+        in_main_thread = is_current_thread_main_thread()
+        if activate_matplotlib and in_main_thread:
             # before every stop check if matplotlib modules were imported inside script code
-            if in_main_thread:
-                self._activate_mpl_if_needed()
+            self._activate_mpl_if_needed()
+
+        while True:
+            with self._main_lock:  # Use lock to check if suspended state changed
+                if info.pydev_state != STATE_SUSPEND or self._finish_debugging_session:
+                    break
 
-            while info.pydev_state == STATE_SUSPEND and not self._finish_debugging_session:
-                if in_main_thread and self.mpl_in_use:
-                    # call input hooks if only matplotlib is in use
-                    self._call_mpl_hook()
+            if in_main_thread and self.mpl_in_use:
+                # call input hooks if only matplotlib is in use
+                self._call_mpl_hook()
 
-                self.process_internal_commands()
-                time.sleep(0.01)
+            self.process_internal_commands()
+            time.sleep(0.01)
 
         self.cancel_async_evaluation(get_current_thread_id(thread), str(id(frame)))
 

src/ptvsd/_vendored/pydevd/tests_python/debugger_unittest.py

@@ -591,6 +591,15 @@ def _ignore_stderr_line(self, line):
                 'from _pydevd_bundle.pydevd_additional_thread_info import set_additional_thread_info',
                 "RuntimeWarning: Parent module '_pydevd_bundle._debug_adapter' not found while handling absolute import",
                 'import json',
+
+                # Issues with Jython and Java 9.
+                'WARNING: Illegal reflective access by org.python.core.PySystemState',
+                'WARNING: Please consider reporting this to the maintainers of org.python.core.PySystemState',
+                'WARNING: An illegal reflective access operation has occurred',
+                'WARNING: Illegal reflective access by jnr.posix.JavaLibCHelper',
+                'WARNING: Please consider reporting this to the maintainers of jnr.posix.JavaLibCHelper',
+                'WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations',
+                'WARNING: All illegal access operations will be denied in a future release',
                 ):
                 if expected in line:
                     return True

src/ptvsd/_vendored/pydevd/tests_python/resources/_debugger_case_multi_threads_stepping.py

@@ -0,0 +1,55 @@
+'''
+After breaking on the thread 1, thread 2 should pause waiting for the event1 to be set,
+so, when we step return on thread 1, the program should finish if all threads are resumed
+or should keep waiting for the thread 2 to run if only thread 1 is resumed.
+'''
+
+import threading
+
+event0 = threading.Event()
+event1 = threading.Event()
+event2 = threading.Event()
+event3 = threading.Event()
+
+
+def _thread1():
+    _event1_set = False
+    _event2_set = False
+
+    while not event0.is_set():
+        event0.wait(timeout=.001)
+
+    event1.set()  # Break thread 1
+    _event1_set = True
+
+    while not event2.is_set():
+        event2.wait(timeout=.001)
+    _event2_set = True  # Note: we can only get here if thread 2 is also released.
+
+    event3.set()
+
+
+def _thread2():
+    event0.set()
+
+    while not event1.is_set():
+        event1.wait(timeout=.001)
+
+    event2.set()
+
+    while not event3.is_set():
+        event3.wait(timeout=.001)
+
+
+if __name__ == '__main__':
+    threads = [
+        threading.Thread(target=_thread1, name='thread1'),
+        threading.Thread(target=_thread2, name='thread2'),
+    ]
+    for t in threads:
+        t.start()
+
+    for t in threads:
+        t.join()
+
+    print('TEST SUCEEDED!')