Make TestBase less stateful.

This fixes zipimporter issues noted in pantsbuild#6282 and is generally desirable
in the absence of severe performance degradation.

Fixes pantsbuild#6282

src/python/pants/backend/python/subsystems/pytest.py

@@ -13,8 +13,7 @@ class PyTest(Subsystem):
   @classmethod
   def register_options(cls, register):
     super(PyTest, cls).register_options(register)
-    # TODO: This is currently bounded below `3.7` due to #6282.
-    register('--requirements', advanced=True, default='pytest>=3.0.7,<3.7',
+    register('--requirements', advanced=True, default='pytest>=3.0.7,<4.0',
              help='Requirements string for the pytest library.')
     register('--timeout-requirements', advanced=True, default='pytest-timeout>=1.2,<1.3',
              help='Requirements string for the pytest-timeout library.')

tests/python/pants_test/backend/python/tasks/test_gather_sources.py

@@ -5,19 +5,14 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
-from builtins import open, str
+from builtins import open
 
-from pex.interpreter import PythonInterpreter
-
-from pants.backend.python.interpreter_cache import PythonInterpreterCache
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.tasks.gather_sources import GatherSources
+from pants.backend.python.tasks.select_interpreter import SelectInterpreter
 from pants.build_graph.files import Files
 from pants.build_graph.resources import Resources
 from pants.source.source_root import SourceRootConfig
-from pants.util.contextutil import temporary_dir
 from pants_test.task_test_base import TaskTestBase
 
 
@@ -107,22 +102,13 @@ def test_gather_files(self):
     self._assert_content_not_in_pex(pex, self.resources)
 
   def _gather_sources(self, target_roots):
-    with temporary_dir() as cache_dir:
-      context = self.context(target_roots=target_roots,
-                             for_subsystems=[PythonSetup, PythonRepos],
-                             options={PythonSetup.options_scope: {'interpreter_cache_dir': cache_dir}})
-
-      # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
-      # to ensure that the interpreter has setuptools and wheel support.
-      interpreter = PythonInterpreter.get()
-      interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
-                                                PythonRepos.global_instance(),
-                                                logger=context.log.debug)
-      interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
-                                            filters=[str(interpreter.identity.requirement)])
-      context.products.get_data(PythonInterpreter, lambda: interpreters[0])
-
-      task = self.create_task(context)
-      task.execute()
-
-      return context.products.get_data(GatherSources.PYTHON_SOURCES)
+    si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si')
+    context = self.context(target_roots=target_roots, for_task_types=[si_task_type])
+
+    si_task = si_task_type(context, os.path.join(self.pants_workdir, 'si'))
+    si_task.execute()
+
+    task = self.create_task(context)
+    task.execute()
+
+    return context.products.get_data(GatherSources.PYTHON_SOURCES)

tests/python/pants_test/backend/python/tasks/test_resolve_requirements.py

@@ -5,18 +5,15 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
-from builtins import str
 
 from pex.interpreter import PythonInterpreter
 
-from pants.backend.python.interpreter_cache import PythonInterpreterCache
 from pants.backend.python.python_requirement import PythonRequirement
-from pants.backend.python.subsystems.python_repos import PythonRepos
-from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
+from pants.backend.python.tasks.select_interpreter import SelectInterpreter
 from pants.base.build_environment import get_buildroot
-from pants.util.contextutil import temporary_dir, temporary_file
+from pants.util.contextutil import temporary_file
 from pants.util.process_handler import subprocess
 from pants_test.task_test_base import TaskTestBase
 
@@ -114,26 +111,16 @@ def _fake_target(self, spec, requirement_strs):
                             requirements=requirements)
 
   def _resolve_requirements(self, target_roots, options=None):
-    with temporary_dir() as cache_dir:
-      options = options or {}
-      options.setdefault(PythonSetup.options_scope, {})['interpreter_cache_dir'] = cache_dir
-      context = self.context(target_roots=target_roots, options=options,
-                             for_subsystems=[PythonSetup, PythonRepos])
-
-      # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
-      # to ensure that the interpreter has setuptools and wheel support.
-      interpreter = PythonInterpreter.get()
-      interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
-                                                PythonRepos.global_instance(),
-                                                logger=context.log.debug)
-      interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
-                                            filters=[str(interpreter.identity.requirement)])
-      context.products.get_data(PythonInterpreter, lambda: interpreters[0])
-
-      task = self.create_task(context)
-      task.execute()
-
-      return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
+    si_task_type = self.synthesize_task_subtype(SelectInterpreter, 'si')
+    context = self.context(target_roots=target_roots, for_task_types=[si_task_type], options=options)
+
+    si_task = si_task_type(context, os.path.join(self.pants_workdir, 'si'))
+    si_task.execute()
+
+    task = self.create_task(context)
+    task.execute()
+
+    return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
 
   def _exercise_module(self, pex, expected_module):
     with temporary_file() as f:

tests/python/pants_test/test_base.py

@@ -352,35 +352,32 @@ def build_root(self):
   def pants_workdir(self):
     return self._pants_workdir()
 
-  @classmethod
   @memoized_method
-  def _build_root(cls):
-    cls.real_build_root = BuildRoot().path
+  def _build_root(self):
+    self.real_build_root = BuildRoot().path
     return os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
 
-  @classmethod
   @memoized_method
-  def _pants_workdir(cls):
-    return os.path.join(cls._build_root(), '.pants.d')
+  def _pants_workdir(self):
+    return os.path.join(self._build_root(), '.pants.d')
 
-  @classmethod
-  def _init_engine(cls):
-    if cls._scheduler is not None:
+  def _init_engine(self):
+    if self._scheduler is not None:
       return
 
     # NB: This uses the long form of initialization because it needs to directly specify
     # `cls.alias_groups` rather than having them be provided by bootstrap options.
     graph_session = EngineInitializer.setup_legacy_graph_extended(
       pants_ignore_patterns=None,
-      workdir=cls._pants_workdir(),
+      workdir=self._pants_workdir(),
       build_file_imports_behavior='allow',
       native=init_native(),
-      build_configuration=cls.build_config(),
+      build_configuration=self.build_config(),
       build_ignore_patterns=None,
     ).new_session()
-    cls._scheduler = graph_session.scheduler_session
-    cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
-        TargetRoots([]), cls._build_root()
+    self._scheduler = graph_session.scheduler_session
+    self._build_graph, self._address_mapper = graph_session.create_build_graph(
+        TargetRoots([]), self._build_root()
       )
 
   @property