Rework configuration to allow generic overrides

Now any loader can be overwritten. Separate better source and loader
concept. The source no longer provides the tox root, but is taken as
construction argument.

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

src/tox/config/cli/env_var.py

@@ -5,7 +5,7 @@
 import os
 from typing import Any, Optional, Tuple, Type
 
-from tox.config.source.ini.convert import StrConvert
+from tox.config.loader.str_convert import StrConvert
 
 CONVERT = StrConvert()
 

src/tox/config/cli/ini.py

@@ -3,12 +3,13 @@
 """
 import logging
 import os
+from configparser import ConfigParser
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple, Type, cast
+from typing import Any, Dict, Optional, Tuple, Type
 
 from appdirs import user_config_dir
 
-from tox.config.source.ini import IniLoader, ToxIni
+from tox.config.loader.ini import IniLoader
 
 DEFAULT_CONFIG_FILE = Path(user_config_dir("tox")) / "config.ini"
 
@@ -29,8 +30,12 @@ def __init__(self) -> None:
         if self.has_config_file:
             self.config_file = self.config_file.absolute()
             try:
-                self.ini = ToxIni(self.config_file)
-                self.has_tox_section = cast(IniLoader, self.ini.core)._section is not None  # noqa
+
+                parser = ConfigParser()
+                with self.config_file.open() as file_handler:
+                    parser.read_file(file_handler)
+                self.has_tox_section = parser.has_section("tox")
+                self.ini: Optional[IniLoader] = IniLoader("tox", parser, overrides=[]) if self.has_tox_section else None
             except Exception as exception:
                 logging.error("failed to read config file %s because %r", config_file, exception)
                 self.has_config_file = None
@@ -41,9 +46,12 @@ def get(self, key: str, of_type: Type[Any]) -> Any:
             result = self._cache[cache_key]
         else:
             try:
-                source = "file"
-                value = self.ini.core.load(key, of_type=of_type, conf=None)
-                result = value, source
+                if self.ini is None:
+                    result = None
+                else:
+                    source = "file"
+                    value = self.ini.load(key, of_type=of_type, conf=None, env_name="tox")
+                    result = value, source
             except KeyError:  # just not found
                 result = None
             except Exception as exception:  # noqa

src/tox/config/cli/parse.py

@@ -9,20 +9,22 @@
 from .parser import Handler, Parsed, ToxParser
 
 Handlers = Dict[str, Handler]
-ParsedOptions = Tuple[Parsed, Handlers]
 
 
-def get_options(*args: str) -> ParsedOptions:
+def get_options(*args: str) -> Tuple[Parsed, Handlers, Sequence[str]]:
+    pos_args: Tuple[str, ...] = ()
     try:  # remove positional arguments passed to parser if specified, they are pulled directly from sys.argv
-        args = args[: args.index("--")]
+        pos_arg_at = args.index("--")
+        pos_args = tuple(args[pos_arg_at + 1 :])
+        args = args[:pos_arg_at]
     except ValueError:
         pass
 
     guess_verbosity = _get_base(args)
     parsed, handlers = _get_all(args)
     if guess_verbosity != parsed.verbosity:
         setup_report(parsed.verbosity, parsed.is_colored)  # pragma: no cover
-    return parsed, handlers
+    return parsed, handlers, pos_args
 
 
 def _get_base(args: Sequence[str]) -> int:
@@ -34,7 +36,7 @@ def _get_base(args: Sequence[str]) -> int:
     return guess_verbosity
 
 
-def _get_all(args: Sequence[str]) -> ParsedOptions:
+def _get_all(args: Sequence[str]) -> Tuple[Parsed, Handlers]:
     """Parse all the options."""
     tox_parser = _get_parser()
     parsed = cast(Parsed, tox_parser.parse_args(args))
@@ -54,5 +56,5 @@ def _get_parser() -> ToxParser:
 
 __all__ = (
     "get_options",
-    "ParsedOptions",
+    "Handlers",
 )

src/tox/config/cli/parser.py

@@ -10,7 +10,7 @@
 from itertools import chain
 from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, cast
 
-from tox.config.source.ini.convert import StrConvert
+from tox.config.loader.str_convert import StrConvert
 from tox.plugin import NAME
 from tox.session.state import State
 
@@ -233,4 +233,5 @@ def _inject_default_cmd(self, args: Sequence[str]) -> Sequence[str]:
     "DEFAULT_VERBOSITY",
     "Parsed",
     "ToxParser",
+    "Handler",
 )

src/tox/config/loader/api.py

@@ -0,0 +1,99 @@
+from abc import abstractmethod
+from argparse import ArgumentTypeError
+from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Set, Type, TypeVar
+
+from tox.plugin.impl import impl
+
+from .convert import Convert
+from .str_convert import StrConvert
+
+if TYPE_CHECKING:
+    from tox.config.cli.parser import ToxParser
+    from tox.config.main import Config
+
+
+class Override:
+    def __init__(self, value: str) -> None:
+        split_at = value.find("=")
+        if split_at == -1:
+            raise ArgumentTypeError(f"override {value} has no = sign in it")
+        key = value[:split_at]
+        ns_at = key.find(".")
+        self.namespace = key[:ns_at]
+        self.key = key[ns_at + 1 :]
+        self.value = value[split_at + 1 :]
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}('{self.namespace}{'.' if self.namespace else ''}{self.key}={self.value}')"
+
+    def __eq__(self, other: Any) -> bool:
+        if type(self) != type(other):
+            return False
+        return (self.namespace, self.key, self.value) == (other.namespace, other.key, other.value)
+
+    def __ne__(self, other: Any) -> bool:
+        return not (self == other)
+
+
+OverrideMap = Mapping[str, List[Override]]
+
+T = TypeVar("T")
+V = TypeVar("V")
+
+
+class Loader(Convert[T]):
+    """Loader loads a configuration value and converts it."""
+
+    def __init__(self, overrides: List[Override]) -> None:
+        self.overrides = {o.key: o for o in overrides}
+
+    @abstractmethod
+    def load_raw(self, key: str, conf: Optional["Config"], env_name: Optional[str]) -> T:
+        """
+        Load the raw object from the config store.
+
+        :param key: the key under what we want the configuration
+        :param env_name: the name of the environment this load is happening for
+        :param conf: the global config object
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def found_keys(self) -> Set[str]:
+        """A list of configuration keys found within the configuration."""
+        raise NotImplementedError
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}"
+
+    def load(self, key: str, of_type: Type[V], conf: Optional["Config"], env_name: Optional[str]) -> V:
+        """
+        Load a value.
+
+        :param key: the key under it lives
+        :param of_type: the type to convert to
+        :param conf: the configuration object of this tox session (needed to manifest the value)
+        :param env_name: env name
+        :return: the converted type
+        """
+        if key in self.overrides:
+            return _STR_CONVERT.to(self.overrides[key].value, of_type)
+        raw = self.load_raw(key, conf, env_name)
+        converted = self.to(raw, of_type)
+        return converted
+
+
+@impl
+def tox_add_option(parser: "ToxParser") -> None:
+    parser.add_argument(
+        "-o",
+        "--override",
+        action="append",
+        type=Override,
+        default=[],
+        dest="override",
+        help="list of configuration override(s)",
+    )
+
+
+_STR_CONVERT = StrConvert()