diff --git a/src/tox/config/cli/env_var.py b/src/tox/config/cli/env_var.py index 578bacc1e..8c7098aee 100644 --- a/src/tox/config/cli/env_var.py +++ b/src/tox/config/cli/env_var.py @@ -5,7 +5,7 @@ import os from typing import Any, Optional, Tuple, Type -from tox.config.source.ini.convert import StrConvert +from tox.config.loader.str_convert import StrConvert CONVERT = StrConvert() diff --git a/src/tox/config/cli/ini.py b/src/tox/config/cli/ini.py index 6cfe5fe4b..67894db89 100644 --- a/src/tox/config/cli/ini.py +++ b/src/tox/config/cli/ini.py @@ -3,12 +3,13 @@ """ import logging import os +from configparser import ConfigParser from pathlib import Path -from typing import Any, Dict, Optional, Tuple, Type, cast +from typing import Any, Dict, Optional, Tuple, Type from appdirs import user_config_dir -from tox.config.source.ini import IniLoader, ToxIni +from tox.config.loader.ini import IniLoader DEFAULT_CONFIG_FILE = Path(user_config_dir("tox")) / "config.ini" @@ -29,8 +30,12 @@ def __init__(self) -> None: if self.has_config_file: self.config_file = self.config_file.absolute() try: - self.ini = ToxIni(self.config_file) - self.has_tox_section = cast(IniLoader, self.ini.core)._section is not None # noqa + + parser = ConfigParser() + with self.config_file.open() as file_handler: + parser.read_file(file_handler) + self.has_tox_section = parser.has_section("tox") + self.ini: Optional[IniLoader] = IniLoader("tox", parser, overrides=[]) if self.has_tox_section else None except Exception as exception: logging.error("failed to read config file %s because %r", config_file, exception) self.has_config_file = None @@ -41,9 +46,12 @@ def get(self, key: str, of_type: Type[Any]) -> Any: result = self._cache[cache_key] else: try: - source = "file" - value = self.ini.core.load(key, of_type=of_type, conf=None) - result = value, source + if self.ini is None: + result = None + else: + source = "file" + value = self.ini.load(key, of_type=of_type, conf=None, env_name="tox") + result = value, source except KeyError: # just not found result = None except Exception as exception: # noqa diff --git a/src/tox/config/cli/parse.py b/src/tox/config/cli/parse.py index 9a134591a..d9249e8ac 100644 --- a/src/tox/config/cli/parse.py +++ b/src/tox/config/cli/parse.py @@ -9,12 +9,14 @@ from .parser import Handler, Parsed, ToxParser Handlers = Dict[str, Handler] -ParsedOptions = Tuple[Parsed, Handlers] -def get_options(*args: str) -> ParsedOptions: +def get_options(*args: str) -> Tuple[Parsed, Handlers, Sequence[str]]: + pos_args: Tuple[str, ...] = () try: # remove positional arguments passed to parser if specified, they are pulled directly from sys.argv - args = args[: args.index("--")] + pos_arg_at = args.index("--") + pos_args = tuple(args[pos_arg_at + 1 :]) + args = args[:pos_arg_at] except ValueError: pass @@ -22,7 +24,7 @@ def get_options(*args: str) -> ParsedOptions: parsed, handlers = _get_all(args) if guess_verbosity != parsed.verbosity: setup_report(parsed.verbosity, parsed.is_colored) # pragma: no cover - return parsed, handlers + return parsed, handlers, pos_args def _get_base(args: Sequence[str]) -> int: @@ -34,7 +36,7 @@ def _get_base(args: Sequence[str]) -> int: return guess_verbosity -def _get_all(args: Sequence[str]) -> ParsedOptions: +def _get_all(args: Sequence[str]) -> Tuple[Parsed, Handlers]: """Parse all the options.""" tox_parser = _get_parser() parsed = cast(Parsed, tox_parser.parse_args(args)) @@ -54,5 +56,5 @@ def _get_parser() -> ToxParser: __all__ = ( "get_options", - "ParsedOptions", + "Handlers", ) diff --git a/src/tox/config/cli/parser.py b/src/tox/config/cli/parser.py index 7a19999be..58deea521 100644 --- a/src/tox/config/cli/parser.py +++ b/src/tox/config/cli/parser.py @@ -10,7 +10,7 @@ from itertools import chain from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, cast -from tox.config.source.ini.convert import StrConvert +from tox.config.loader.str_convert import StrConvert from tox.plugin import NAME from tox.session.state import State @@ -233,4 +233,5 @@ def _inject_default_cmd(self, args: Sequence[str]) -> Sequence[str]: "DEFAULT_VERBOSITY", "Parsed", "ToxParser", + "Handler", ) diff --git a/src/tox/config/core.py b/src/tox/config/core.py deleted file mode 100644 index 4dfba8716..000000000 --- a/src/tox/config/core.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Define configuration options that are part of the core tox configurations""" -from pathlib import Path -from typing import cast - -from tox.config.sets import ConfigSet -from tox.config.source.api import EnvList -from tox.plugin.impl import impl - - -@impl -def tox_add_core_config(core: ConfigSet) -> None: - core.add_config( - keys=["work_dir", "toxworkdir"], - of_type=Path, - # here we pin to .tox4 to be able to use in parallel with v3 until final release - default=lambda conf, _: cast(Path, conf.core["tox_root"]) / ".tox4", - desc="working directory", - ) - core.add_config( - keys=["temp_dir"], - of_type=Path, - default=lambda conf, _: cast(Path, conf.core["tox_root"]) / ".temp", - desc="temporary directory cleaned at start", - ) - core.add_config( - keys=["env_list", "envlist"], - of_type=EnvList, - default=EnvList([]), - desc="define environments to automatically run", - ) - core.add_config( - keys=["skip_missing_interpreters"], - of_type=bool, - default=True, - desc="skip missing interpreters", - ) diff --git a/src/tox/config/loader/__init__.py b/src/tox/config/loader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/tox/config/loader/api.py b/src/tox/config/loader/api.py new file mode 100644 index 000000000..f2340f012 --- /dev/null +++ b/src/tox/config/loader/api.py @@ -0,0 +1,99 @@ +from abc import abstractmethod +from argparse import ArgumentTypeError +from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Set, Type, TypeVar + +from tox.plugin.impl import impl + +from .convert import Convert +from .str_convert import StrConvert + +if TYPE_CHECKING: + from tox.config.cli.parser import ToxParser + from tox.config.main import Config + + +class Override: + def __init__(self, value: str) -> None: + split_at = value.find("=") + if split_at == -1: + raise ArgumentTypeError(f"override {value} has no = sign in it") + key = value[:split_at] + ns_at = key.find(".") + self.namespace = key[:ns_at] + self.key = key[ns_at + 1 :] + self.value = value[split_at + 1 :] + + def __repr__(self) -> str: + return f"{self.__class__.__name__}('{self.namespace}{'.' if self.namespace else ''}{self.key}={self.value}')" + + def __eq__(self, other: Any) -> bool: + if type(self) != type(other): + return False + return (self.namespace, self.key, self.value) == (other.namespace, other.key, other.value) + + def __ne__(self, other: Any) -> bool: + return not (self == other) + + +OverrideMap = Mapping[str, List[Override]] + +T = TypeVar("T") +V = TypeVar("V") + + +class Loader(Convert[T]): + """Loader loads a configuration value and converts it.""" + + def __init__(self, overrides: List[Override]) -> None: + self.overrides = {o.key: o for o in overrides} + + @abstractmethod + def load_raw(self, key: str, conf: Optional["Config"], env_name: Optional[str]) -> T: + """ + Load the raw object from the config store. + + :param key: the key under what we want the configuration + :param env_name: the name of the environment this load is happening for + :param conf: the global config object + """ + raise NotImplementedError + + @abstractmethod + def found_keys(self) -> Set[str]: + """A list of configuration keys found within the configuration.""" + raise NotImplementedError + + def __repr__(self) -> str: + return f"{type(self).__name__}" + + def load(self, key: str, of_type: Type[V], conf: Optional["Config"], env_name: Optional[str]) -> V: + """ + Load a value. + + :param key: the key under it lives + :param of_type: the type to convert to + :param conf: the configuration object of this tox session (needed to manifest the value) + :param env_name: env name + :return: the converted type + """ + if key in self.overrides: + return _STR_CONVERT.to(self.overrides[key].value, of_type) + raw = self.load_raw(key, conf, env_name) + converted = self.to(raw, of_type) + return converted + + +@impl +def tox_add_option(parser: "ToxParser") -> None: + parser.add_argument( + "-o", + "--override", + action="append", + type=Override, + default=[], + dest="override", + help="list of configuration override(s)", + ) + + +_STR_CONVERT = StrConvert() diff --git a/src/tox/config/source/api.py b/src/tox/config/loader/convert.py similarity index 52% rename from src/tox/config/source/api.py rename to src/tox/config/loader/convert.py index 82e70b2a7..ee0274a0b 100644 --- a/src/tox/config/source/api.py +++ b/src/tox/config/loader/convert.py @@ -3,72 +3,16 @@ from collections import OrderedDict from enum import Enum from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generic, - Iterator, - List, - Optional, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) - -from tox.execute.request import shell_cmd +from typing import Any, Dict, Generic, Iterator, List, Set, Tuple, Type, TypeVar, Union, cast if sys.version_info >= (3, 8): from typing import Literal else: - from typing_extensions import Literal # noqa - -if TYPE_CHECKING: - from tox.config.main import Config - from tox.config.sets import ConfigSet -_NO_MAPPING = object() - - -class Command: - def __init__(self, args: List[str]) -> None: - self.ignore_exit_code = args[0] == "-" - self.args = args[1:] if self.ignore_exit_code else args - - def __repr__(self) -> str: - return f"{type(self).__name__}(args={self.args!r})" - - def __eq__(self, other: Any) -> bool: - return type(self) == type(other) and self.args == other.args - - def __ne__(self, other: Any) -> bool: - return not (self == other) - - @property - def shell(self) -> str: - return shell_cmd(self.args) - - -class EnvList: - def __init__(self, envs: Sequence[str]) -> None: - self.envs = list(OrderedDict((e, None) for e in envs).keys()) - - def __repr__(self) -> str: - return "{}(envs={!r})".format(type(self).__name__, ",".join(self.envs)) - - def __eq__(self, other: Any) -> bool: - return type(self) == type(other) and self.envs == other.envs - - def __ne__(self, other: Any) -> bool: - return not (self == other) - - def __iter__(self) -> Iterator[str]: - return iter(self.envs) + from typing_extensions import Literal +from ..types import Command, EnvList +_NO_MAPPING = object() T = TypeVar("T") V = TypeVar("V") @@ -80,7 +24,7 @@ def to(self, raw: T, of_type: Type[V]) -> V: from_module = getattr(of_type, "__module__", None) if from_module in ("typing", "typing_extensions"): - return self._to_typing(raw, of_type) + return self._to_typing(raw, of_type) # type: ignore[return-value] elif issubclass(of_type, Path): return self.to_path(raw) # type: ignore[return-value] elif issubclass(of_type, bool): @@ -170,75 +114,3 @@ def to_env_list(value: T) -> EnvList: @abstractmethod def to_bool(value: T) -> bool: raise NotImplementedError - - -class Loader(Convert[T]): - """Loader loads a configuration value and converts it.""" - - def __init__(self, name: Optional[str], namespace: str) -> None: - """ - Create a loader. - - :param name: name of this loader: ``None`` for core, otherwise the tox environment name - :param name: the namespace under the name exists within the config - """ - self.name = name - self.namespace = namespace - - def load(self, key: str, of_type: Type[V], conf: Optional["Config"]) -> V: - """ - Load a value. - - :param key: the key under it lives - :param of_type: the type to convert to - :param conf: the configuration object of this tox session (needed to manifest the value) - :return: the converted type - """ - raw = self._load_raw(key, conf) - converted = self.to(raw, of_type) - return converted - - @abstractmethod - def setup_with_conf(self, conf: "ConfigSet") -> None: - """Notifies the loader when the global configuration object has been constructed""" - raise NotImplementedError - - def make_package_conf(self) -> None: - """Notifies the loader that this is a package configuration.""" - - @abstractmethod - def _load_raw(self, key: str, conf: Optional["Config"]) -> T: - """ - Load the raw object from the config store. - - :param key: the key under what we want the configuration - :param conf: the global config object - """ - raise NotImplementedError - - @abstractmethod - def found_keys(self) -> Set[str]: - """A list of configuration keys found within the configuration.""" - raise NotImplementedError - - -class Source(ABC): - """ - Source is able to return a configuration value (for either the core or per environment source). - """ - - def __init__(self, core: Loader[Any]) -> None: - self.core = core - - @abstractmethod - def envs(self, core_conf: "ConfigSet") -> Iterator[str]: - raise NotImplementedError - - @abstractmethod - def __getitem__(self, item: str) -> Loader[Any]: - raise NotImplementedError - - @property - @abstractmethod - def tox_root(self) -> Path: - raise NotImplementedError diff --git a/src/tox/config/loader/ini/__init__.py b/src/tox/config/loader/ini/__init__.py new file mode 100644 index 000000000..f07cf9e48 --- /dev/null +++ b/src/tox/config/loader/ini/__init__.py @@ -0,0 +1,57 @@ +from configparser import ConfigParser, SectionProxy +from copy import deepcopy +from typing import Any, List, Optional, Set, TypeVar + +from tox.config.loader.api import Loader, Override +from tox.config.loader.ini.factor import filter_for_env +from tox.config.loader.ini.replace import replace +from tox.config.loader.str_convert import StrConvert +from tox.config.main import Config + +V = TypeVar("V") + + +class IniLoader(StrConvert, Loader[str]): + """Load configuration from an ini section (ini file is a string to string dictionary)""" + + def __init__( + self, + section: str, + parser: ConfigParser, + overrides: List[Override], + ) -> None: + self._section: SectionProxy = parser[section] + self._parser = parser + super().__init__(overrides) + + def load_raw(self, key: str, conf: Optional[Config], env_name: Optional[str]) -> str: + value = self._section[key] + collapsed_newlines = value.replace("\\\r", "").replace("\\\n", "") # collapse explicit line splits + replace_executed = replace(collapsed_newlines, conf, env_name, self) # do replacements + factor_selected = filter_for_env(replace_executed, env_name) # select matching factors + # extend factors + return factor_selected + + def found_keys(self) -> Set[str]: + return set(self._section.keys()) + + def get_section(self, name: str) -> Optional[SectionProxy]: + # needed for non tox environment replacements + if self._parser.has_section(name): + return self._parser[name] + return None + + def __deepcopy__(self, memo: Any) -> "IniLoader": + # python < 3.7 cannot copy config parser + result: IniLoader = self.__class__.__new__(self.__class__) + memo[id(self)] = result + for k, v in self.__dict__.items(): + if k != "_section": + value = deepcopy(v, memo=memo) # noqa + else: + value = v + setattr(result, k, value) + return result + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(section={self._section}, overrides={self.overrides!r})" diff --git a/src/tox/config/source/ini/factor.py b/src/tox/config/loader/ini/factor.py similarity index 100% rename from src/tox/config/source/ini/factor.py rename to src/tox/config/loader/ini/factor.py diff --git a/src/tox/config/source/ini/replace.py b/src/tox/config/loader/ini/replace.py similarity index 88% rename from src/tox/config/source/ini/replace.py rename to src/tox/config/loader/ini/replace.py index c4160f31c..78b919867 100644 --- a/src/tox/config/source/ini/replace.py +++ b/src/tox/config/loader/ini/replace.py @@ -3,18 +3,16 @@ """ import os import re -import sys from configparser import SectionProxy -from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Iterator, List, Optional, Sequence, Tuple, Union +from tox.config.loader.stringify import stringify from tox.config.main import Config from tox.config.sets import ConfigSet from tox.execute.request import shell_cmd -from .stringify import stringify - if TYPE_CHECKING: - from tox.config.source.ini import IniLoader + from tox.config.loader.ini import IniLoader CORE_PREFIX = "tox" BASE_TEST_ENV = "testenv" @@ -70,7 +68,9 @@ def _replace_match( if of_type == "env": replace_value: Optional[str] = replace_env(args) elif of_type == "posargs": - replace_value = replace_posarg(args) + if conf is None: + raise RuntimeError("no configuration yet") + replace_value = replace_posargs(args, conf.pos_args) else: replace_value = replace_reference(conf, current_env, loader, value) if replace_value is None: @@ -117,7 +117,7 @@ def replace_reference( default = settings["default"] if default is not None: return default - except Exception: # noqa # ignore errors - but don't replace them + except Exception as exc: # noqa # ignore errors - but don't replace them pass # we should raise here - but need to implement escaping factor conditionals # raise ValueError(f"could not replace {value} from {current_env}") @@ -134,7 +134,7 @@ def _config_value_sources( # if we have an env name specified take only from there if env is not None: if conf is not None and env in conf: - yield conf[env] + yield conf.get_env(env) return # if we have a section name specified take only from there @@ -144,7 +144,7 @@ def _config_value_sources( if conf is not None: yield conf.core return - value = loader.section_loader(section) + value = loader.get_section(section) if value is not None: yield value return @@ -153,12 +153,12 @@ def _config_value_sources( if conf is not None: yield conf.core if current_env is not None: - yield conf[current_env] + yield conf.get_env(current_env) -def replace_posarg(args: List[str]) -> str: +def replace_posargs(args: List[str], pos_args: Sequence[str]) -> str: try: - replace_value = shell_cmd(sys.argv[sys.argv.index("--") + 1 :]) + replace_value = shell_cmd(pos_args) except ValueError: replace_value = args[0] if args else "" return replace_value diff --git a/src/tox/config/source/ini/convert.py b/src/tox/config/loader/str_convert.py similarity index 93% rename from src/tox/config/source/ini/convert.py rename to src/tox/config/loader/str_convert.py index 66a563ae3..98ef51071 100644 --- a/src/tox/config/source/ini/convert.py +++ b/src/tox/config/loader/str_convert.py @@ -4,7 +4,8 @@ from pathlib import Path from typing import Iterator, Tuple -from tox.config.source.api import Command, Convert, EnvList +from tox.config.loader.convert import Convert +from tox.config.types import Command, EnvList class StrConvert(Convert[str]): @@ -52,7 +53,7 @@ def to_command(value: str) -> Command: @staticmethod def to_env_list(value: str) -> EnvList: - from tox.config.source.ini.factor import extend_factors + from tox.config.loader.ini.factor import extend_factors elements = list(chain.from_iterable(extend_factors(expr) for expr in value.split("\n"))) return EnvList(elements) diff --git a/src/tox/config/source/ini/stringify.py b/src/tox/config/loader/stringify.py similarity index 86% rename from src/tox/config/source/ini/stringify.py rename to src/tox/config/loader/stringify.py index e71c7908c..64ed03c5a 100644 --- a/src/tox/config/source/ini/stringify.py +++ b/src/tox/config/loader/stringify.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Any, Mapping, Sequence, Set, Tuple -from tox.config.source.api import Command, EnvList +from tox.config.types import Command, EnvList def stringify(value: Any) -> Tuple[str, bool]: @@ -26,11 +26,6 @@ def stringify(value: Any) -> Tuple[str, bool]: return "\n".join(e for e in value.envs), True if isinstance(value, Command): return value.shell, True - - from tox.config.source.ini import IniLoader - - if isinstance(value, IniLoader): - return value.section_name or "", False if value.__repr__ != value.__str__: # use the value return str(value), False raise TypeError(f"type {type(value).__name__} with value {value!r}") diff --git a/src/tox/config/main.py b/src/tox/config/main.py index 0c146b76b..143f46e7c 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -1,68 +1,65 @@ -from collections import OrderedDict +from collections import OrderedDict, defaultdict from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List +from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence -from tox.plugin.impl import impl +from tox.config.loader.api import Override, OverrideMap +from tox.config.source import Source -from .override import Override -from .sets import ConfigSet -from .source.api import Source +from .sets import ConfigSet, CoreConfigSet -if TYPE_CHECKING: - from tox.config.cli.parser import ToxParser +class Config: + def __init__( + self, + config_source: Source, + overrides: List[Override], + root: Path, + pos_args: Sequence[str], + ) -> None: + self.pos_args = pos_args + self._root = root -@impl -def tox_add_option(parser: "ToxParser") -> None: - parser.add_argument( - "-o", - "--override", - action="append", - type=Override, - default=[], - dest="override", - help="list of configuration override(s)", - ) - + self._overrides: OverrideMap = defaultdict(list) + for override in overrides: + self._overrides[override.namespace].append(override) -class Config: - def __init__(self, config_source: Source, overrides: List[Override]) -> None: - self.overrides = overrides self._src = config_source - self.core = self._setup_core() - self._env_names = list(self._src.envs(self.core)) - self._envs: Dict[str, ConfigSet] = OrderedDict() + self._env_to_set: Dict[str, ConfigSet] = OrderedDict() + self._core_set: Optional[CoreConfigSet] = None self.register_config_set: Callable[[str], Any] = lambda x: None - def _setup_core(self) -> ConfigSet: - core = ConfigSet(self._src.core, self) - core.add_config( - keys=["tox_root", "toxinidir"], - of_type=Path, - default=self._src.tox_root, - desc="the root directory (where the configuration file is found)", - ) + @property + def core(self) -> CoreConfigSet: + if self._core_set is not None: + return self._core_set + core = CoreConfigSet(self, self._root) + for loader in self._src.get_core(self._overrides): + core.add_loader(loader) + from tox.plugin.manager import MANAGER MANAGER.tox_add_core_config(core) + self._core_set = core return core - def __getitem__(self, item: str) -> ConfigSet: + def get_env(self, item: str, package: bool = False) -> ConfigSet: try: - return self._envs[item] + return self._env_to_set[item] except KeyError: - env = ConfigSet(self._src[item], self) - self._envs[item] = env + env = ConfigSet(self, item) + self._env_to_set[item] = env + for loader in self._src.get_env_loaders(item, self._overrides, package, env): + env.add_loader(loader) # whenever we load a new configuration we need build a tox environment which process defines the valid # configuration values self.register_config_set(item) return env def __iter__(self) -> Iterator[str]: - return iter(self._env_names) + return self._src.envs(self.core) def __repr__(self) -> str: return f"{type(self).__name__}(config_source={self._src!r})" def __contains__(self, item: str) -> bool: - return item in self._env_names + return any(name for name in self if name == item) diff --git a/src/tox/config/of_type.py b/src/tox/config/of_type.py new file mode 100644 index 000000000..6148a180c --- /dev/null +++ b/src/tox/config/of_type.py @@ -0,0 +1,108 @@ +""" +Group together configuration values that belong together (such as base tox configuration, tox environment configs) +""" +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import TYPE_CHECKING, Callable, Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union, cast + +from tox.config.loader.api import Loader + +if TYPE_CHECKING: + from tox.config.main import Config # pragma: no cover + + +T = TypeVar("T") +V = TypeVar("V") + + +class ConfigDefinition(ABC, Generic[T]): + """Abstract base class for configuration definitions""" + + def __init__(self, keys: Iterable[str], desc: str, env_name: Optional[str]) -> None: + self.keys = keys + self.desc = desc + self.env_name = env_name + + @abstractmethod + def __call__(self, conf: "Config", key: Optional[str], loaders: List[Loader[T]]) -> T: + raise NotImplementedError + + +class ConfigConstantDefinition(ConfigDefinition[T]): + """A configuration definition whose value is defined upfront (such as the tox environment name)""" + + def __init__( + self, + keys: Iterable[str], + desc: str, + env_name: Optional[str], + value: Union[Callable[[], T], T], + ) -> None: + super().__init__(keys, desc, env_name) + self.value = value + + def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]]) -> T: + if callable(self.value): + value = self.value() + else: + value = self.value + return value + + +_PLACE_HOLDER = object() + + +class ConfigDynamicDefinition(ConfigDefinition[T]): + """A configuration definition that comes from a source (such as in memory, an ini file, a toml file, etc.)""" + + def __init__( + self, + keys: Iterable[str], + desc: str, + env_name: Optional[str], + of_type: Type[T], + default: Union[Callable[["Config", Optional[str]], T], T], + post_process: Optional[Callable[[T, "Config"], T]] = None, + ) -> None: + super().__init__(keys, desc, env_name) + self.of_type = of_type + self.default = default + self.post_process = post_process + self._cache: Union[object, T] = _PLACE_HOLDER + + def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]]) -> T: + if self._cache is _PLACE_HOLDER: + found = False + for key in self.keys: + for loader in loaders: + try: + value = loader.load(key, self.of_type, conf, self.env_name) + found = True + except KeyError: + continue + break + if found: + break + else: + value = self.default(conf, self.env_name) if callable(self.default) else self.default + if self.post_process is not None: + value = self.post_process(value, conf) # noqa + self._cache = value + return cast(T, self._cache) + + def __deepcopy__(self, memo: Dict[int, "ConfigDynamicDefinition[T]"]) -> "ConfigDynamicDefinition[T]": + # we should not copy the place holder as our checks would break + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + if k != "_cache" and v is _PLACE_HOLDER: + value = deepcopy(v, memo=memo) # noqa + else: + value = v + setattr(result, k, value) + return cast(ConfigDynamicDefinition[T], result) + + def __repr__(self) -> str: + values = ((k, v) for k, v in vars(self).items() if k != "post_process" and v is not None) + return f"{type(self).__name__}({', '.join('{}={}'.format(k, v) for k,v in values)})" diff --git a/src/tox/config/override.py b/src/tox/config/override.py deleted file mode 100644 index ef9154b21..000000000 --- a/src/tox/config/override.py +++ /dev/null @@ -1,27 +0,0 @@ -from argparse import ArgumentTypeError -from typing import Any - - -class Override: - def __init__(self, value: str) -> None: - split_at = value.find("=") - if split_at == -1: - raise ArgumentTypeError(f"override {value} has no = sign in it") - key = value[:split_at] - ns_at = key.find(".") - self.namespace = key[:ns_at] - self.key = key[ns_at + 1 :] - self.value = value[split_at + 1 :] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.namespace}{'.' if self.namespace else ''}{self.key}={self.value}')" - - def __eq__(self, other: Any) -> bool: - return type(self) == type(other) and (self.namespace, self.key, self.value) == ( - other.namespace, - other.key, - other.value, - ) - - def __ne__(self, other: Any) -> bool: - return not (self == other) diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index ac4527bce..11009937d 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -1,18 +1,12 @@ -""" -Group together configuration values that belong together (such as base tox configuration, tox environment configs) -""" -from abc import ABC, abstractmethod from collections import OrderedDict -from copy import deepcopy from pathlib import Path from typing import ( TYPE_CHECKING, Any, Callable, Dict, - Generic, - Iterable, Iterator, + List, Optional, Sequence, Set, @@ -22,120 +16,28 @@ cast, ) -from tox.config.source.api import Loader +from .of_type import ConfigConstantDefinition, ConfigDefinition, ConfigDynamicDefinition +from .types import EnvList if TYPE_CHECKING: - from tox.config.main import Config # pragma: no cover + from tox.config.loader.api import Loader + from tox.config.main import Config - -T = TypeVar("T") V = TypeVar("V") -class ConfigDefinition(ABC, Generic[T]): - """Abstract base class for configuration definitions""" - - def __init__(self, keys: Iterable[str], desc: str) -> None: - self.keys = keys - self.desc = desc - - @abstractmethod - def __call__(self, src: Loader[T], conf: "Config") -> T: - raise NotImplementedError - - -class ConfigConstantDefinition(ConfigDefinition[T]): - """A configuration definition whose value is defined upfront (such as the tox environment name)""" - - def __init__(self, keys: Iterable[str], desc: str, value: Union[Callable[[], T], T]) -> None: - super().__init__(keys, desc) - self.value = value - - def __call__(self, src: Loader[T], conf: "Config") -> T: - if callable(self.value): - value = self.value() - else: - value = self.value - return value - - -_PLACE_HOLDER = object() - - -class ConfigDynamicDefinition(ConfigDefinition[T]): - """A configuration definition that comes from a source (such as in memory, an ini file, a toml file, etc.)""" - - def __init__( - self, - keys: Iterable[str], - of_type: Type[T], - default: Union[Callable[["Config", Optional[str]], T], T], - desc: str, - post_process: Optional[Callable[[T, "Config"], T]] = None, - ) -> None: - super().__init__(keys, desc) - self.of_type = of_type - self.default = default - self.post_process = post_process - self._cache: Union[object, T] = _PLACE_HOLDER - - def __call__(self, src: Loader[T], conf: "Config") -> T: - if self._cache is _PLACE_HOLDER: - for key in self.keys: - override = next((o for o in conf.overrides if o.namespace == src.namespace and o.key == key), None) - if override is not None: - from tox.config.source.ini.convert import StrConvert - - value = StrConvert().to(override.value, self.of_type) - - # relative override paths are relative to tox root unless the tox root itself, which is cwd - if isinstance(value, Path) and not value.is_absolute(): - if key in ["tox_root", "toxinidir"]: - value = value.absolute() # type: ignore[assignment] - else: - value = cast(Path, conf.core["tox_root"]) / value # type: ignore[assignment] - break - else: - for key in self.keys: - try: - value = src.load(key, self.of_type, conf) - except KeyError: - continue - break - else: - value = self.default(conf, src.name) if callable(self.default) else self.default - if self.post_process is not None: - value = self.post_process(value, conf) # noqa - self._cache = value - return cast(T, self._cache) - - def __deepcopy__(self, memo: Dict[int, "ConfigDynamicDefinition[T]"]) -> "ConfigDynamicDefinition[T]": - # we should not copy the place holder as our checks would break - cls = self.__class__ - result = cls.__new__(cls) - memo[id(self)] = result - for k, v in self.__dict__.items(): - if k != "_cache" and v is _PLACE_HOLDER: - value = deepcopy(v, memo=memo) # noqa - else: - value = v - setattr(result, k, value) - return cast(ConfigDynamicDefinition[T], result) - - def __repr__(self) -> str: - values = ((k, v) for k, v in vars(self).items() if k != "post_process" and v is not None) - return f"{type(self).__name__}({', '.join('{}={}'.format(k, v) for k,v in values)})" - - class ConfigSet: """A set of configuration that belong together (such as a tox environment settings, core tox settings)""" - def __init__(self, raw: Loader[Any], conf: "Config"): - self._raw = raw - self._defined: Dict[str, ConfigDefinition[Any]] = {} + def __init__(self, conf: "Config", name: Optional[str]): + self.name = name self._conf = conf + self._loaders: List[Loader[Any]] = [] + self._defined: Dict[str, ConfigDefinition[Any]] = {} self._keys: Dict[str, None] = OrderedDict() - self._raw.setup_with_conf(self) + + def add_loader(self, loader: "Loader[Any]") -> None: + self._loaders.append(loader) def add_config( self, @@ -156,19 +58,16 @@ def add_config( if isinstance(defined, ConfigDynamicDefinition): return defined raise TypeError(f"{keys} already defined with differing type {type(defined).__name__}") - definition = ConfigDynamicDefinition(keys_, of_type, default, desc, post_process) + definition = ConfigDynamicDefinition(keys_, desc, self.name, of_type, default, post_process) self._add_conf(keys_, definition) return definition def add_constant(self, keys: Sequence[str], desc: str, value: V) -> ConfigConstantDefinition[V]: keys_ = self._make_keys(keys) - definition = ConfigConstantDefinition(keys_, desc, value) + definition = ConfigConstantDefinition(keys_, desc, self.name, value) self._add_conf(keys, definition) return definition - def make_package_conf(self) -> None: - self._raw.make_package_conf() - @staticmethod def _make_keys(keys: Union[str, Sequence[str]]) -> Sequence[str]: return (keys,) if isinstance(keys, str) else keys @@ -178,20 +77,56 @@ def _add_conf(self, keys: Union[str, Sequence[str]], definition: ConfigDefinitio for key in keys: self._defined[key] = definition - @property - def name(self) -> Optional[str]: - return self._raw.name - def __getitem__(self, item: str) -> Any: config_definition = self._defined[item] - return config_definition(self._raw, self._conf) + return config_definition(self._conf, item, self._loaders) def __repr__(self) -> str: - return "{}(raw={!r}, conf={!r})".format(type(self).__name__, self._raw, self._conf) + values = (v for v in (f"name={self.name!r}" if self.name else "", f"loaders={self._loaders!r}") if v) + return f"{self.__class__.__name__}({', '.join(values)})" def __iter__(self) -> Iterator[str]: return iter(self._keys.keys()) def unused(self) -> Set[str]: """Return a list of keys present in the config source but not used""" - return self._raw.found_keys() - set(self._defined.keys()) + found = set() + for loader in self._loaders: + found.update(loader.found_keys()) + return found - set(self._defined.keys()) + + +class CoreConfigSet(ConfigSet): + def __init__(self, conf: "Config", root: Path) -> None: + super().__init__(conf, name=None) + self.add_config( + keys=["tox_root", "toxinidir"], + of_type=Path, + default=root, + desc="the root directory (where the configuration file is found)", + ) + self.add_config( + keys=["work_dir", "toxworkdir"], + of_type=Path, + # here we pin to .tox4 to be able to use in parallel with v3 until final release + default=lambda conf, _: cast(Path, self["tox_root"]) / ".tox4", + desc="working directory", + ) + self.add_config( + keys=["temp_dir"], + of_type=Path, + default=lambda conf, _: cast(Path, self["tox_root"]) / ".temp", + desc="temporary directory cleaned at start", + ) + self.add_config( + keys=["env_list", "envlist"], + of_type=EnvList, + default=EnvList([]), + desc="define environments to automatically run", + ) + self.add_config( + keys=["skip_missing_interpreters"], + of_type=bool, + default=True, + desc="skip missing interpreters", + ) diff --git a/src/tox/config/source/__init__.py b/src/tox/config/source/__init__.py index e69de29bb..128c5ec35 100644 --- a/src/tox/config/source/__init__.py +++ b/src/tox/config/source/__init__.py @@ -0,0 +1,30 @@ +"""Sources.""" +from abc import ABC, abstractmethod +from typing import Any, Iterator + +from tox.config.loader.api import Loader, OverrideMap + +from ..sets import ConfigSet, CoreConfigSet + + +class Source(ABC): + """ + Source is able to return a configuration value (for either the core or per environment source). + """ + + @abstractmethod + def get_core(self, override_map: OverrideMap) -> Iterator[Loader[Any]]: + """Return the core loader from this source.""" + raise NotImplementedError + + @abstractmethod + def get_env_loaders( + self, env_name: str, override_map: OverrideMap, package: bool, conf: ConfigSet + ) -> Iterator[Loader[Any]]: + """Return the load for this environment.""" + raise NotImplementedError + + @abstractmethod + def envs(self, core_conf: "CoreConfigSet") -> Iterator[str]: + """Return a list of environments defined within this source""" + raise NotImplementedError diff --git a/src/tox/config/source/ini/__init__.py b/src/tox/config/source/ini/__init__.py deleted file mode 100644 index e423a54ef..000000000 --- a/src/tox/config/source/ini/__init__.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Load """ -from configparser import ConfigParser, SectionProxy -from copy import deepcopy -from itertools import chain -from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Type, TypeVar - -from tox.config.main import Config -from tox.config.sets import ConfigSet -from tox.config.source.api import Loader, Source -from tox.config.source.ini.convert import StrConvert -from tox.config.source.ini.factor import filter_for_env, find_envs -from tox.config.source.ini.replace import BASE_TEST_ENV, CORE_PREFIX, replace - -TEST_ENV_PREFIX = f"{BASE_TEST_ENV}:" - -V = TypeVar("V") - - -class IniLoader(StrConvert, Loader[str]): - """Load configuration from an ini section (ini file is a string to string dictionary)""" - - def to(self, raw: str, of_type: Type[V]) -> V: - if of_type == IniLoader: - return self._src[raw] # type: ignore[return-value] - return super(IniLoader, self).to(raw, of_type) - - def __init__( - self, - section: Optional[SectionProxy], - src: "ToxIni", - name: Optional[str], - default_base: List["IniLoader"], - section_loader: Callable[[str], Optional[SectionProxy]], - namespace: str, - ) -> None: - super().__init__(name, namespace) - self._section: Optional[SectionProxy] = section - self._src: ToxIni = src - self._default_base = default_base - self._base: List[IniLoader] = [] - self.section_loader = section_loader - - def __deepcopy__(self, memo: Any) -> "IniLoader": - # python < 3.7 cannot copy config parser - result: IniLoader = self.__class__.__new__(self.__class__) - memo[id(self)] = result - for k, v in self.__dict__.items(): - if k != "_section": - value = deepcopy(v, memo=memo) # noqa - else: - value = v - setattr(result, k, value) - return result - - def setup_with_conf(self, conf: ConfigSet) -> None: - if self.name is None: - return # no inheritance for the base tox environment - # allow environment inheritance - conf.add_config( - keys="base", - of_type=List[IniLoader], - default=self._default_base, - desc="inherit missing keys from these sections", - # builder=lambda raw: self._src[raw], - ) - self._base = conf["base"] - - def make_package_conf(self) -> None: - """no inheritance please if this is a packaging env""" - self._base = [] - - def __repr__(self) -> str: - return "{}(section={}, src={!r})".format( - type(self).__name__, - self._section.name if self._section else self.name, - self._src, - ) - - def _load_raw(self, key: str, conf: Optional[Config], as_name: Optional[str] = None) -> str: - for candidate in self.loaders: - if as_name is None and candidate.name == "": - as_name = self.name - try: - return candidate._load_raw_from(as_name, conf, key) - except KeyError: - continue - else: - raise KeyError - - def _load_raw_from(self, as_name: Optional[str], conf: Optional["Config"], key: str) -> str: - if as_name is None: - as_name = self.name - if self._section is None: - raise KeyError(key) - value = self._section[key] - collapsed_newlines = value.replace("\\\r", "").replace("\\\n", "") # collapse explicit line splits - replace_executed = replace(collapsed_newlines, conf, as_name, self) # do replacements - factor_selected = filter_for_env(replace_executed, as_name) # select matching factors - # extend factors - return factor_selected - - def get_value(self, section: str, key: str) -> str: - section_proxy = self.section_loader(section) - if section_proxy is None: - raise KeyError(section) - return section_proxy[key] - - @property - def loaders(self) -> Iterator["IniLoader"]: - yield self - yield from self._base - - def found_keys(self) -> Set[str]: - result: Set[str] = set() - for candidate in self.loaders: - if candidate._section is not None: - result.update(candidate._section.keys()) - return result - - @property - def section_name(self) -> Optional[str]: - if self._section is None: - return None - return self._section.name - - -class ToxIni(Source): - """Configuration sourced from a ini file (such as tox.ini)""" - - def __init__(self, path: Path) -> None: - self._path = path - - self._parser = ConfigParser() - with self._path.open() as file_handler: - self._parser.read_file(file_handler) - core = IniLoader( - section=self._get_section(CORE_PREFIX), - src=self, - name=None, - default_base=[], - section_loader=self._get_section, - namespace=CORE_PREFIX, - ) - super().__init__(core=core) - self._envs: Dict[str, IniLoader] = {} - - def _get_section(self, key: str) -> Optional[SectionProxy]: - if self._parser.has_section(key): - return self._parser[key] - return None - - def __deepcopy__(self, memo: Dict[int, Any]) -> "ToxIni": - # python < 3.7 cannot copy config parser - result: ToxIni = self.__class__.__new__(self.__class__) - memo[id(self)] = result - for k, v in self.__dict__.items(): - if k != "_parser": - value = deepcopy(v, memo=memo) # noqa - else: - value = v - setattr(result, k, value) - return result - - @property - def tox_root(self) -> Path: - return self._path.parent.absolute() - - def envs(self, core_config: ConfigSet) -> Iterator[str]: - seen = set() - for name in self._discover_tox_envs(core_config): - if name not in seen: - seen.add(name) - yield name - - def __getitem__(self, item: str) -> "IniLoader": - key = f"{TEST_ENV_PREFIX}{item}" - return self.get_section(key, item) - - def get_section(self, item: str, name: str) -> "IniLoader": - try: - return self._envs[item] - except KeyError: - base = [] if item == BASE_TEST_ENV else [self.get_section(BASE_TEST_ENV, "")] - loader = IniLoader( - section=self._get_section(item), - src=self, - name=name, - default_base=base, - section_loader=self._get_section, - namespace=item, - ) - self._envs[item] = loader - return loader - - def _discover_tox_envs(self, core_config: ConfigSet) -> Iterator[str]: - explicit = list(core_config["env_list"]) - yield from explicit - known_factors = None - for section in self._parser.sections(): - if section.startswith(BASE_TEST_ENV): - is_base_section = section == BASE_TEST_ENV - name = BASE_TEST_ENV if is_base_section else section[len(TEST_ENV_PREFIX) :] - if not is_base_section: - yield name - if known_factors is None: - known_factors = set(chain.from_iterable(e.split("-") for e in explicit)) - yield from self._discover_from_section(section, known_factors) - - def _discover_from_section(self, section: str, known_factors: Set[str]) -> Iterator[str]: - for key in self._parser[section]: - value = self._parser[section].get(key) - if value: - for env in find_envs(value): - if env not in known_factors: - yield env - - def __repr__(self) -> str: - return f"{type(self).__name__}(path={self._path})" - - -__all__ = ( - "ToxIni", - "IniLoader", - "filter_for_env", - "find_envs", -) diff --git a/src/tox/config/source/tox_ini.py b/src/tox/config/source/tox_ini.py new file mode 100644 index 000000000..3d1a44b11 --- /dev/null +++ b/src/tox/config/source/tox_ini.py @@ -0,0 +1,135 @@ +"""Load """ +from configparser import ConfigParser, SectionProxy +from copy import deepcopy +from itertools import chain +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Set + +from tox.config.loader.ini.factor import find_envs + +from ..loader.api import OverrideMap +from ..loader.ini import IniLoader +from ..loader.ini.replace import BASE_TEST_ENV, CORE_PREFIX +from ..sets import ConfigSet +from . import Source + +TEST_ENV_PREFIX = f"{BASE_TEST_ENV}:" + + +class ToxIni(Source): + """Configuration sourced from a ini file (such as tox.ini)""" + + def __init__(self, path: Path) -> None: + self._path = path + self._parser = ConfigParser() + with self._path.open() as file_handler: + self._parser.read_file(file_handler) + + super().__init__() + self._envs: Dict[Optional[str], List[IniLoader]] = {} + + def get_core(self, override_map: OverrideMap) -> Iterator[IniLoader]: + if None in self._envs: + yield from self._envs[None] + return + core = [] + if self._parser.has_section(CORE_PREFIX): + core.append( + IniLoader( + section=CORE_PREFIX, + parser=self._parser, + overrides=override_map.get(CORE_PREFIX, []), + ) + ) + self._envs[None] = core + yield from core + + def get_env_loaders( + self, env_name: str, override_map: OverrideMap, package: bool, conf: ConfigSet + ) -> Iterator[IniLoader]: + section = f"{TEST_ENV_PREFIX}{env_name}" + try: + yield from self._envs[section] + except KeyError: + loaders: List[IniLoader] = [] + self._envs[section] = loaders + + if self._parser.has_section(section): + loader = IniLoader( + section=section, + parser=self._parser, + overrides=override_map.get(section, []), + ) + yield loader + loaders.append(loader) + + if package is False: + conf.add_config( # base may be override within the testenv:py section + keys="base", + of_type=List[str], + desc="inherit missing keys from these sections", + default=[BASE_TEST_ENV], + ) + for base in conf["base"]: + for section in (base, f"{TEST_ENV_PREFIX}{base}"): + if self._parser.has_section(section): + loader = IniLoader( + section=section, + parser=self._parser, + overrides=override_map.get(section, []), + ) + yield loader + loaders.append(loader) + break + + def envs(self, core_config: ConfigSet) -> Iterator[str]: + seen = set() + for name in self._discover_tox_envs(core_config): + if name not in seen: + seen.add(name) + yield name + + def _get_section(self, key: str) -> Optional[SectionProxy]: + if self._parser.has_section(key): + return self._parser[key] + return None + + def __deepcopy__(self, memo: Dict[int, Any]) -> "ToxIni": + # python < 3.7 cannot copy config parser + result: ToxIni = self.__class__.__new__(self.__class__) + memo[id(self)] = result + for k, v in self.__dict__.items(): + if k != "_parser": + value = deepcopy(v, memo=memo) # noqa + else: + value = v + setattr(result, k, value) + return result + + def _discover_tox_envs(self, core_config: ConfigSet) -> Iterator[str]: + explicit = list(core_config["env_list"]) + yield from explicit + known_factors = None + for section in self._parser.sections(): + if section.startswith(BASE_TEST_ENV): + is_base_section = section == BASE_TEST_ENV + name = BASE_TEST_ENV if is_base_section else section[len(TEST_ENV_PREFIX) :] + if not is_base_section: + yield name + if known_factors is None: + known_factors = set(chain.from_iterable(e.split("-") for e in explicit)) + yield from self._discover_from_section(section, known_factors) + + def _discover_from_section(self, section: str, known_factors: Set[str]) -> Iterator[str]: + for key in self._parser[section]: + value = self._parser[section].get(key) + if value: + for env in find_envs(value): + if env not in known_factors: + yield env + + def __repr__(self) -> str: + return f"{type(self).__name__}(path={self._path})" + + +__all__ = ("ToxIni",) diff --git a/src/tox/config/types.py b/src/tox/config/types.py new file mode 100644 index 000000000..0b8af9de1 --- /dev/null +++ b/src/tox/config/types.py @@ -0,0 +1,46 @@ +from collections import OrderedDict +from typing import Any, Iterator, List, Sequence + +from tox.execute.request import shell_cmd + + +class Command: + def __init__(self, args: List[str]) -> None: + self.ignore_exit_code = args[0] == "-" + self.args = args[1:] if self.ignore_exit_code else args + + def __repr__(self) -> str: + return f"{type(self).__name__}(args={self.args!r})" + + def __eq__(self, other: Any) -> bool: + return type(self) == type(other) and self.args == other.args + + def __ne__(self, other: Any) -> bool: + return not (self == other) + + @property + def shell(self) -> str: + return shell_cmd(self.args) + + +class EnvList: + def __init__(self, envs: Sequence[str]) -> None: + self.envs = list(OrderedDict((e, None) for e in envs).keys()) + + def __repr__(self) -> str: + return "{}(envs={!r})".format(type(self).__name__, ",".join(self.envs)) + + def __eq__(self, other: Any) -> bool: + return type(self) == type(other) and self.envs == other.envs + + def __ne__(self, other: Any) -> bool: + return not (self == other) + + def __iter__(self) -> Iterator[str]: + return iter(self.envs) + + +__all__ = ( + "Command", + "EnvList", +) diff --git a/src/tox/plugin/manager.py b/src/tox/plugin/manager.py index 08f63da95..390b224a2 100644 --- a/src/tox/plugin/manager.py +++ b/src/tox/plugin/manager.py @@ -4,9 +4,8 @@ import pluggy from tox import provision -from tox.config import core as core_config -from tox.config import main as main_config from tox.config.cli.parser import ToxParser +from tox.config.loader import api as loader_api from tox.config.sets import ConfigSet from tox.session import state from tox.session.cmd import list_env, show_config, version_flag @@ -25,9 +24,8 @@ def __init__(self) -> None: self.manager.add_hookspecs(spec) internal_plugins = ( - main_config, + loader_api, provision, - core_config, runner, dev, sdist, diff --git a/src/tox/pytest.py b/src/tox/pytest.py index c7dce93eb..e82314a53 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -20,6 +20,7 @@ from _pytest.python import Function import tox.run +from tox.config.loader.api import Override from tox.config.main import Config from tox.execute.api import Outcome from tox.execute.request import shell_cmd @@ -124,8 +125,16 @@ def structure(self) -> Dict[str, Any]: into[file_name] = (dir_path / file_name).read_text() return result - def config(self) -> Config: - return tox.run.make_config(self.path, []) + def config( + self, + overrides: Optional[List[Override]] = None, + pos_args: Optional[Sequence[str]] = None, + ) -> Config: + return tox.run.make_config( + path=self.path, + overrides=[] if overrides is None else overrides, + pos_args=[] if pos_args is None else pos_args, + ) def run(self, *args: str) -> "ToxRunOutcome": cur_dir = os.getcwd() diff --git a/src/tox/run.py b/src/tox/run.py index 177b0759b..74c8b8f84 100644 --- a/src/tox/run.py +++ b/src/tox/run.py @@ -6,9 +6,9 @@ from typing import List, Optional, Sequence, cast from tox.config.cli.parse import get_options +from tox.config.loader.api import Override from tox.config.main import Config -from tox.config.override import Override -from tox.config.source.ini import ToxIni +from tox.config.source.tox_ini import ToxIni from tox.report import HandledError from tox.session.state import State @@ -41,16 +41,16 @@ def setup_state(args: Sequence[str]) -> State: """Setup the state object of this run.""" start = datetime.now() # parse CLI arguments - options = get_options(*args) - options[0].start = start + parsed, handlers, pos_args = get_options(*args) + parsed.start = start # parse configuration file - config = make_config(Path().cwd().absolute(), options[0].override) + config = make_config(Path().cwd().absolute(), parsed.override, pos_args) # build tox environment config objects - state = State(config, options, args) + state = State(config, (parsed, handlers), args) return state -def make_config(path: Path, overrides: List[Override]) -> Config: +def make_config(path: Path, overrides: List[Override], pos_args: Sequence[str]) -> Config: """Make a tox configuration object.""" # for now only tox.ini supported folder = path @@ -58,7 +58,7 @@ def make_config(path: Path, overrides: List[Override]) -> Config: tox_ini = folder / "tox.ini" if tox_ini.exists() and tox_ini.is_file(): ini_loader = ToxIni(tox_ini) - return Config(ini_loader, overrides) + return Config(ini_loader, overrides, tox_ini.parent, pos_args) if folder.parent == folder: break folder = folder.parent diff --git a/src/tox/session/cmd/run/single.py b/src/tox/session/cmd/run/single.py index 7d51b7425..32af6f9eb 100644 --- a/src/tox/session/cmd/run/single.py +++ b/src/tox/session/cmd/run/single.py @@ -3,7 +3,7 @@ """ from typing import List, cast -from tox.config.source.api import Command +from tox.config.types import Command from tox.execute.api import Outcome from tox.tox_env.api import ToxEnv from tox.tox_env.errors import Recreate diff --git a/src/tox/session/cmd/show_config.py b/src/tox/session/cmd/show_config.py index fcce0e1b2..abd83dca8 100644 --- a/src/tox/session/cmd/show_config.py +++ b/src/tox/session/cmd/show_config.py @@ -5,8 +5,8 @@ from textwrap import indent from tox.config.cli.parser import ToxParser +from tox.config.loader.stringify import stringify from tox.config.sets import ConfigSet -from tox.config.source.ini.stringify import stringify from tox.plugin.impl import impl from tox.session.common import env_list_flag from tox.session.state import State @@ -40,6 +40,8 @@ def print_conf(conf: ConfigSet) -> None: for key in conf: value = conf[key] as_str, multi_line = stringify(value) + if multi_line and "\n" not in as_str: + multi_line = False if multi_line and as_str.strip(): print(f"{key} =\n{indent(as_str, prefix=' ')}") else: diff --git a/src/tox/session/common.py b/src/tox/session/common.py index e4203f5d4..330562ff5 100644 --- a/src/tox/session/common.py +++ b/src/tox/session/common.py @@ -1,7 +1,7 @@ from argparse import Action, ArgumentParser, Namespace from typing import Any, List, Optional, Sequence, Union, cast -from tox.config.source.ini.convert import StrConvert +from tox.config.loader.str_convert import StrConvert def env_list_flag(parser: ArgumentParser) -> None: diff --git a/src/tox/session/state.py b/src/tox/session/state.py index 7cbbe324d..141481dea 100644 --- a/src/tox/session/state.py +++ b/src/tox/session/state.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Sequence, Set, cast +from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Sequence, Set, Tuple, cast from tox.config.main import Config from tox.config.sets import ConfigSet @@ -8,15 +8,16 @@ from tox.tox_env.runner import RunToxEnv if TYPE_CHECKING: - from tox.config.cli.parse import ParsedOptions - from tox.config.cli.parser import ToxParser + + from tox.config.cli.parse import Handlers + from tox.config.cli.parser import Parsed, ToxParser class State: def __init__( self, conf: Config, - opt_parse: "ParsedOptions", + opt_parse: Tuple["Parsed", "Handlers"], args: Sequence[str], ) -> None: self.conf = conf @@ -45,7 +46,7 @@ def tox_env(self, name: str) -> RunToxEnv: tox_env = self._run_env.get(name) if tox_env is not None: return tox_env - env_conf = self.conf[name] + env_conf = self.conf.get_env(name) tox_env = self._build_run_env(env_conf, name) self._run_env[name] = tox_env return tox_env @@ -99,8 +100,7 @@ def _get_package_env(self, packager: str, name: str) -> PackageToxEnv: package_type = REGISTER.package(packager) self._pkg_env_discovered.add(name) - pkg_conf = self.conf[name] - pkg_conf.make_package_conf() + pkg_conf = self.conf.get_env(name, package=True) pkg_tox_env = package_type(pkg_conf, self.conf.core, self.options) self._pkg_env[name] = pkg_tox_env return pkg_tox_env diff --git a/src/tox/tox_env/runner.py b/src/tox/tox_env/runner.py index 9e3e337bf..173334550 100644 --- a/src/tox/tox_env/runner.py +++ b/src/tox/tox_env/runner.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Generator, List, Optional, Tuple, cast from tox.config.sets import ConfigSet -from tox.config.source.api import Command, EnvList +from tox.config.types import Command, EnvList from .api import ToxEnv from .package import PackageToxEnv diff --git a/tests/config/cli/test_cli_env_var.py b/tests/config/cli/test_cli_env_var.py index da061fb14..0b92b726b 100644 --- a/tests/config/cli/test_cli_env_var.py +++ b/tests/config/cli/test_cli_env_var.py @@ -3,23 +3,23 @@ import pytest from tox.config.cli.parse import get_options -from tox.config.override import Override +from tox.config.loader.api import Override from tox.pytest import CaptureFixture, LogCaptureFixture, MonkeyPatch from tox.session.state import State def test_verbose(monkeypatch: MonkeyPatch) -> None: - parsed, _ = get_options("-v", "-v") + parsed, _, __ = get_options("-v", "-v") assert parsed.verbosity == 4 def test_verbose_compound(monkeypatch: MonkeyPatch) -> None: - parsed, _ = get_options("-vv") + parsed, _, __ = get_options("-vv") assert parsed.verbosity == 4 def test_verbose_no_test(monkeypatch: MonkeyPatch) -> None: - parsed, _ = get_options("--notest", "-vv", "--runner", "virtualenv") + parsed, _, __ = get_options("--notest", "-vv", "--runner", "virtualenv") assert vars(parsed) == { "colored": "no", "verbose": 4, @@ -47,7 +47,7 @@ def test_env_var_exhaustive_parallel_values( monkeypatch.setenv("TOX_PARALLEL_LIVE", "no") monkeypatch.setenv("TOX_OVERRIDE", "a=b\nc=d") - parsed, handlers = get_options() + parsed, handlers, _ = get_options() assert vars(parsed) == { "colored": "no", "verbose": 5, @@ -82,7 +82,7 @@ def test_bad_env_var( ) -> None: monkeypatch.setenv("TOX_VERBOSE", "should-be-number") monkeypatch.setenv("TOX_QUIET", "1.00") - parsed, _ = get_options() + parsed, _, __ = get_options() assert parsed.verbose == 2 assert parsed.quiet == 0 assert parsed.verbosity == 2 diff --git a/tests/config/cli/test_cli_ini.py b/tests/config/cli/test_cli_ini.py index f35a6b5d6..72724331f 100644 --- a/tests/config/cli/test_cli_ini.py +++ b/tests/config/cli/test_cli_ini.py @@ -7,7 +7,7 @@ import pytest from tox.config.cli.parse import get_options -from tox.config.override import Override +from tox.config.loader.api import Override from tox.pytest import CaptureFixture, LogCaptureFixture, MonkeyPatch from tox.session.state import State @@ -54,7 +54,7 @@ def empty_ini(tmp_path: Path, monkeypatch: MonkeyPatch) -> Path: def test_ini_empty(empty_ini: Path, core_handlers: Dict[str, Callable[[State], int]]) -> None: - parsed, handlers = get_options() + parsed, handlers, _ = get_options() assert vars(parsed) == { "colored": "no", "verbose": 2, @@ -71,7 +71,7 @@ def test_ini_empty(empty_ini: Path, core_handlers: Dict[str, Callable[[State], i def test_ini_exhaustive_parallel_values(exhaustive_ini: Path, core_handlers: Dict[str, Callable[[State], int]]) -> None: - parsed, handlers = get_options() + parsed, handlers, _ = get_options() assert vars(parsed) == { "colored": "yes", "verbose": 5, @@ -101,7 +101,7 @@ def test_ini_help(exhaustive_ini: Path, capsys: CaptureFixture) -> None: def test_bad_cli_ini(tmp_path: Path, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture) -> None: caplog.set_level(logging.WARNING) monkeypatch.setenv("TOX_CONFIG_FILE", str(tmp_path)) - parsed, _ = get_options() + parsed, __, _ = get_options() msg = ( "PermissionError(13, 'Permission denied')" if sys.platform == "win32" @@ -136,7 +136,7 @@ def test_bad_option_cli_ini( ), ) monkeypatch.setenv("TOX_CONFIG_FILE", str(to)) - parsed, _ = get_options() + parsed, _, __ = get_options() assert caplog.messages == [ "{} key verbose as type failed with {}".format( to, diff --git a/tests/config/ini/replace/conftest.py b/tests/config/ini/replace/conftest.py index bb4b52a9a..0bac6fbe8 100644 --- a/tests/config/ini/replace/conftest.py +++ b/tests/config/ini/replace/conftest.py @@ -1,25 +1,24 @@ -from contextlib import contextmanager -from typing import Callable, ContextManager, Iterator, Optional +import sys +from typing import List, Optional, cast import pytest -from tox.config.main import Config from tox.pytest import ToxProjectCreator +if sys.version_info > (3, 7): + from typing import Protocol +else: + from typing_extensions import Protocol -class Result: - def __init__(self) -> None: - self.config: Optional[Config] = None - self.val: Optional[str] = None - -ReplaceOne = Callable[[str], ContextManager[Result]] +class ReplaceOne(Protocol): + def __call__(self, conf: str, pos_args: Optional[List[str]] = None) -> str: + ... @pytest.fixture def replace_one(tox_project: ToxProjectCreator) -> ReplaceOne: - @contextmanager - def example(conf: str) -> Iterator[Result]: + def example(conf: str, pos_args: Optional[List[str]] = None) -> str: project = tox_project( { "tox.ini": f""" @@ -32,12 +31,9 @@ def example(conf: str) -> Iterator[Result]: """, }, ) - - result = Result() - yield result - result.config = project.config() - env_config = result.config["a"] + config = project.config(pos_args=pos_args) + env_config = config.get_env("a") env_config.add_config(keys="env", of_type=str, default="bad", desc="env") - result.val = env_config["env"] + return cast(str, env_config["env"]) - return example + return example # noqa diff --git a/tests/config/ini/replace/test_do_not_replace.py b/tests/config/ini/replace/test_do_not_replace.py index b11e85c07..7b3f32ce6 100644 --- a/tests/config/ini/replace/test_do_not_replace.py +++ b/tests/config/ini/replace/test_do_not_replace.py @@ -19,6 +19,5 @@ ) def test_do_not_replace(replace_one: ReplaceOne, start: str, end: str) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one(start) as result: - pass - assert result.val == end + value = replace_one(start) + assert value == end diff --git a/tests/config/ini/replace/test_replace_env_var.py b/tests/config/ini/replace/test_replace_env_var.py index df10024e5..6020735cf 100644 --- a/tests/config/ini/replace/test_replace_env_var.py +++ b/tests/config/ini/replace/test_replace_env_var.py @@ -4,28 +4,29 @@ def test_replace_env_set(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{env:MAGIC}") as result: - monkeypatch.setenv("MAGIC", "something good") - assert result.val == "something good" + monkeypatch.setenv("MAGIC", "something good") + result = replace_one("{env:MAGIC}") + + assert result == "something good" def test_replace_env_missing(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{env:MAGIC}") as result: - monkeypatch.delenv("MAGIC", raising=False) - assert result.val == "" + monkeypatch.delenv("MAGIC", raising=False) + result = replace_one("{env:MAGIC}") + assert result == "" def test_replace_env_missing_default(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{env:MAGIC:def}") as result: - monkeypatch.delenv("MAGIC", raising=False) - assert result.val == "def" + monkeypatch.delenv("MAGIC", raising=False) + result = replace_one("{env:MAGIC:def}") + assert result == "def" def test_replace_env_missing_default_from_env(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{env:MAGIC:{env:MAGIC_DEFAULT}}") as result: - monkeypatch.delenv("MAGIC", raising=False) - monkeypatch.setenv("MAGIC_DEFAULT", "yes") - assert result.val == "yes" + monkeypatch.delenv("MAGIC", raising=False) + monkeypatch.setenv("MAGIC_DEFAULT", "yes") + result = replace_one("{env:MAGIC:{env:MAGIC_DEFAULT}}") + assert result == "yes" diff --git a/tests/config/ini/replace/test_replace_posargs.py b/tests/config/ini/replace/test_replace_posargs.py index e770c9068..1806adfdc 100644 --- a/tests/config/ini/replace/test_replace_posargs.py +++ b/tests/config/ini/replace/test_replace_posargs.py @@ -6,21 +6,21 @@ def test_replace_pos_args_empty_sys_argv(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{posargs}") as result: - monkeypatch.setattr(sys, "argv", []) - assert result.val == "" + monkeypatch.setattr(sys, "argv", []) + result = replace_one("{posargs}", []) + assert result == "" def test_replace_pos_args_extra_sys_argv(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{posargs}") as result: - monkeypatch.setattr(sys, "argv", [sys.executable, "magic"]) - assert result.val == "" + monkeypatch.setattr(sys, "argv", [sys.executable, "magic"]) + result = replace_one("{posargs}", []) + + assert result == "" def test_replace_pos_args(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: """If we have a factor that is not specified within the core env-list then that's also an environment""" - with replace_one("{posargs}") as result: - monkeypatch.setattr(sys, "argv", [sys.executable, "magic", "--", "ok", "what", " yes "]) + result = replace_one("{posargs}", ["ok", "what", " yes "]) quote = '"' if sys.platform == "win32" else "'" - assert result.val == f"ok what {quote} yes {quote}" + assert result == f"ok what {quote} yes {quote}" diff --git a/tests/config/ini/replace/test_replace_tox_env.py b/tests/config/ini/replace/test_replace_tox_env.py index 320f1fa56..7734b2fc6 100644 --- a/tests/config/ini/replace/test_replace_tox_env.py +++ b/tests/config/ini/replace/test_replace_tox_env.py @@ -13,7 +13,7 @@ def example(tox_project: ToxProjectCreator) -> EnvConfigCreator: def func(conf: str) -> ConfigSet: project = tox_project({"tox.ini": f"""[tox]\nenv_list = a\n[testenv]\n{conf}\n"""}) config = project.config() - env_config = config["a"] + env_config = config.get_env("a") return env_config return func diff --git a/tests/config/ini/test_factor.py b/tests/config/ini/test_factor.py index 7f54e876e..aa2208260 100644 --- a/tests/config/ini/test_factor.py +++ b/tests/config/ini/test_factor.py @@ -3,7 +3,7 @@ import pytest -from tox.config.source.ini import filter_for_env, find_envs +from tox.config.loader.ini.factor import filter_for_env, find_envs from tox.pytest import ToxProjectCreator @@ -117,7 +117,7 @@ def test_factor_config(tox_project: ToxProjectCreator) -> None: config = project.config() assert list(config) == ["py36-django15", "py36-django16", "py37-django15", "py37-django16"] for env in config.core["env_list"]: - env_config = config[env] + env_config = config.get_env(env) env_config.add_config(keys="deps", of_type=List[str], default=[], desc="deps", overwrite=True) deps = env_config["deps"] assert "pytest" in deps diff --git a/tests/config/ini/test_values.py b/tests/config/ini/test_values.py index 6b9b72c07..143d83cf6 100644 --- a/tests/config/ini/test_values.py +++ b/tests/config/ini/test_values.py @@ -1,4 +1,4 @@ -from tox.config.source.api import Command +from tox.config.types import Command from tox.pytest import ToxProjectCreator diff --git a/tests/config/test_main.py b/tests/config/test_main.py index 38de32930..59202a253 100644 --- a/tests/config/test_main.py +++ b/tests/config/test_main.py @@ -6,7 +6,6 @@ from tox.config.main import Config from tox.config.sets import ConfigSet -from tox.config.source.ini import IniLoader from tox.pytest import ToxProject, ToxProjectCreator @@ -31,11 +30,10 @@ def test_empty_conf_tox_envs(empty_config: Config) -> None: def test_empty_conf_get(empty_config: Config) -> None: - result = empty_config["magic"] + result = empty_config.get_env("magic") assert isinstance(result, ConfigSet) loaders = result["base"] - assert len(loaders) == 1 - assert isinstance(loaders[0], IniLoader) + assert loaders == ["testenv"] def test_config_some_envs(tox_project: ToxProjectCreator) -> None: @@ -51,7 +49,7 @@ def test_config_some_envs(tox_project: ToxProjectCreator) -> None: tox_env_keys = list(config) assert tox_env_keys == ["py38", "py37", "other", "magic"] - config_set = config["py38"] + config_set = config.get_env("py38") assert repr(config_set) assert isinstance(config_set, ConfigSet) assert list(config_set) == ["base"] @@ -63,7 +61,7 @@ def test_config_some_envs(tox_project: ToxProjectCreator) -> None: @pytest.fixture(name="conf_builder") def _conf_builder(tox_project: ToxProjectCreator) -> ConfBuilder: def _make(conf_str: str) -> ConfigSet: - return tox_project({"tox.ini": f"[tox]\nenvlist=py39\n[testenv]\n{conf_str}"}).config()["py39"] + return tox_project({"tox.ini": f"[tox]\nenvlist=py39\n[testenv]\n{conf_str}"}).config().get_env("py39") return _make diff --git a/tests/session/cmd/test_show_config_env.py b/tests/session/cmd/test_show_config_env.py index 0c52894f7..3102bbad4 100644 --- a/tests/session/cmd/test_show_config_env.py +++ b/tests/session/cmd/test_show_config_env.py @@ -20,8 +20,7 @@ def test_list_empty(tox_project: ToxProjectCreator) -> None: skip_missing_interpreters = True min_version = {__version__} provision_tox_env = .tox - requires = - tox>={__version__} + requires = tox>={__version__} no_package = False """, ).lstrip() diff --git a/tests/tox_env/test_show_config.py b/tests/tox_env/test_show_config.py index 75d171489..e63f3ca7a 100644 --- a/tests/tox_env/test_show_config.py +++ b/tests/tox_env/test_show_config.py @@ -34,18 +34,16 @@ def test_show_config_default_run_env(tox_project: ToxProjectCreator, monkeypatch tox_root = {path} work_dir = {path}{sep}\.tox4 temp_dir = {path}{sep}\.temp - env_list = - {name} + env_list = {name} skip_missing_interpreters = True min_version = {version} provision_tox_env = \.tox - requires = - tox>={version} + requires = tox>={version} no_package = False \[testenv:{name}\] type = VirtualEnvRunner - base = + base = testenv runner = virtualenv env_name = {name} env_dir = {path}{sep}\.tox4{sep}{name} @@ -56,8 +54,7 @@ def test_show_config_default_run_env(tox_project: ToxProjectCreator, monkeypatch pass_env = {pass_env_str} description = - commands = - magic + commands = magic commands_pre = commands_post = change_dir = {path} @@ -69,8 +66,7 @@ def test_show_config_default_run_env(tox_project: ToxProjectCreator, monkeypatch package_tox_env_type = virtualenv-pep-517-sdist package_env = \.package extras = - base_python = - {name} + base_python = {name} env_site_packages_dir = {path}{sep}\.tox4{sep}{name}{sep}.* env_python = {path}{sep}\.tox4{sep}{name}{sep}.* deps =