diff --git a/openfisca_core/errors.py b/openfisca_core/errors.py index d0d137daed..72dfcef8ee 100644 --- a/openfisca_core/errors.py +++ b/openfisca_core/errors.py @@ -32,7 +32,7 @@ def __init__(self, variable_name, tax_benefit_system): ]) self.message = message self.variable_name = variable_name - Exception.__init__(self, self.message.encode('utf-8')) + Exception.__init__(self, self.message) class SituationParsingError(Exception): diff --git a/openfisca_core/scripts/run_test.py b/openfisca_core/scripts/run_test.py index cb224b5a22..26b787ec53 100644 --- a/openfisca_core/scripts/run_test.py +++ b/openfisca_core/scripts/run_test.py @@ -23,7 +23,4 @@ def main(parser): } paths = [os.path.abspath(path) for path in args.path] - tests_ok = run_tests(tax_benefit_system, paths, options) - - if not tests_ok: - sys.exit(1) + sys.exit(run_tests(tax_benefit_system, paths, options)) diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 7859e877b0..a5a5af8648 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -1,25 +1,16 @@ # -*- coding: utf-8 -*- -""" -A module to run openfisca yaml tests -""" - -from builtins import str - -import glob -import os -import sys -import unittest import logging +import sys +import os import traceback -import nose +import pytest from openfisca_core.tools import assert_near from openfisca_core.simulation_builder import SimulationBuilder from openfisca_core.errors import SituationParsingError, VariableNotFound - log = logging.getLogger(__name__) @@ -45,33 +36,6 @@ def import_yaml(): _tax_benefit_system_cache = {} -# Exposed methods - -def generate_tests(tax_benefit_system, paths, options = None): - """ - Generates a lazy iterator of all the YAML tests contained in a file or a directory. - - :parameters: Same as :meth:`run_tests` - - :return: a generator of YAML tests - - """ - - if isinstance(paths, str): - paths = [paths] - - if options is None: - options = {} - - for path in paths: - if os.path.isdir(path): - for test in _generate_tests_from_directory(tax_benefit_system, path, options): - yield test - else: - for test in _generate_tests_from_file(tax_benefit_system, path, options): - yield test - - def run_tests(tax_benefit_system, paths, options = None): """ Runs all the YAML tests contained in a file or a directory. @@ -97,94 +61,162 @@ def run_tests(tax_benefit_system, paths, options = None): +-------------------------------+-----------+-------------------------------------------+ """ - argv = sys.argv[:1] # Nose crashes if it gets any unexpected argument. - if options is None: - options = {} + argv = ["--capture", "no"] if options.get('pdb'): argv.append('--pdb') - if options.get('verbose'): - argv.append('--nocapture') # Do not capture output when verbose mode is activated - - argv.append('--nologcapture') # Do not capture logs so that the user can define a log level + if isinstance(paths, str): + paths = [paths] - return nose.run( - # The suite argument must be a lambda for nose to run the tests lazily - suite = lambda: generate_tests(tax_benefit_system, paths, options), - argv = argv, - ) + if options is None: + options = {} + return pytest.main([*argv, *paths] if True else paths, plugins = [OpenFiscaPlugin(tax_benefit_system, options)]) -# Internal methods -def _generate_tests_from_file(tax_benefit_system, path_to_file, options): - filename = os.path.splitext(os.path.basename(path_to_file))[0] - name_filter = options.get('name_filter') - verbose = options.get('verbose') +class YamlFile(pytest.File): - tests = _parse_test_file(tax_benefit_system, path_to_file, options) + def __init__(self, path, parent, tax_benefit_system, options): + super(YamlFile, self).__init__(path, parent) + self.tax_benefit_system = tax_benefit_system + self.options = options - for _test_index, (simulation, test) in enumerate(tests, 1): - if name_filter is not None and name_filter not in filename \ - and name_filter not in (test.get('name', '')) \ - and name_filter not in (test.get('keywords', [])): - continue + def collect(self): + try: + tests = yaml.load(self.fspath.open(), Loader = Loader) + except (yaml.scanner.ScannerError, TypeError): + message = os.linesep.join([ + traceback.format_exc(), + f"'{self.fspath}' is not a valid YAML file. Check the stack trace above for more details.", + ]) + raise ValueError(message) - keywords = test.get('keywords', []) - title = "{}: {}{} - {}".format( - os.path.basename(path_to_file), - '[{}] '.format(', '.join(keywords)) if keywords else '', - test.get('name'), - test.get('period'), + if not isinstance(tests, list): + tests = [tests] + for test in tests: + if not self.should_ignore(test): + yield YamlItem(self.fspath.basename, self, self.tax_benefit_system, test, self.options) + + def should_ignore(self, test): + name_filter = self.options.get('name_filter') + return ( + name_filter is not None + and name_filter not in os.path.splitext(self.fspath.basename)[0] + and name_filter not in test.get('name', '') + and name_filter not in test.get('keywords', []) ) - test.update({'options': options}) - - def check(): - try: - _run_test(simulation, test) - except Exception: - log.error(title) - raise - finally: - if verbose: - print("Computation log:") # noqa T001 - simulation.tracer.print_computation_log() - - yield unittest.FunctionTestCase(check) -def _generate_tests_from_directory(tax_benefit_system, path_to_dir, options): - yaml_paths = glob.glob(os.path.join(path_to_dir, "*.yaml")) - subdirectories = glob.glob(os.path.join(path_to_dir, "*/")) +class YamlItem(pytest.Item): + def __init__(self, name, parent, baseline_tax_benefit_system, test, options): + super(YamlItem, self).__init__(name, parent) + self.baseline_tax_benefit_system = baseline_tax_benefit_system + self.options = options + self.test = test + self.simulation = None + self.tax_benefit_system = None - for yaml_path in yaml_paths: - for test in _generate_tests_from_file(tax_benefit_system, yaml_path, options): - yield test + def parse_test(self): + name = self.test.get('name', '') + if not self.test.get('output'): + raise ValueError("Missing key 'output' in test '{}' in file '{}'".format(name, self.fspath)) - for subdirectory in subdirectories: - for test in _generate_tests_from_directory(tax_benefit_system, subdirectory, options): - yield test + if not TEST_KEYWORDS.issuperset(self.test.keys()): + unexpected_keys = set(self.test.keys()).difference(TEST_KEYWORDS) + raise ValueError("Unexpected keys {} in test '{}' in file '{}'".format(unexpected_keys, name, self.fspath)) + self.tax_benefit_system = _get_tax_benefit_system(self.baseline_tax_benefit_system, self.test.get('reforms', []), self.test.get('extensions', [])) -def _parse_test_file(tax_benefit_system, yaml_path, options): - with open(yaml_path) as yaml_file: try: - tests = yaml.load(yaml_file, Loader = Loader) - except (yaml.scanner.ScannerError, TypeError): + builder = SimulationBuilder() + input = self.test.get('input', {}) + period = self.test.get('period') + verbose = self.options.get('verbose') + builder.set_default_period(period) + self.simulation = builder.build_from_dict(self.tax_benefit_system, input) + self.simulation.trace = verbose + except SituationParsingError as error: message = os.linesep.join([ traceback.format_exc(), - "'{}'' is not a valid YAML file. Check the stack trace above for more details.".format(yaml_path), + str(error.error), + os.linesep, + "Could not parse situation described in test '{}' in YAML file '{}'. Check the stack trace above for more details.".format(name, self.fspath), ]) raise ValueError(message) + except Exception as e: + error_message = os.linesep.join([str(e), '', f"Unexpected error raised while parsing '{self.fspath}'"]) + raise ValueError(error_message).with_traceback(sys.exc_info()[2]) from e # Keep the stack trace from the root error + + def runtest(self): + self.parse_test() + verbose = self.options.get('verbose') + try: + self.check_output() + finally: + if verbose: + print("Computation log:") # noqa T001 + self.simulation.tracer.print_computation_log() + + def check_output(self): + tax_benefit_system = self.tax_benefit_system + output = self.test.get('output') + + if output is None: + return + for key, expected_value in output.items(): + if tax_benefit_system.variables.get(key): # If key is a variable + self.check_variable(key, expected_value, self.test.get('period')) + elif self.simulation.populations.get(key): # If key is an entity singular + for variable_name, value in expected_value.items(): + self.check_variable(variable_name, value, self.test.get('period')) + else: + population = self.simulation.get_population(plural = key) + if population is not None: # If key is an entity plural + for instance_id, instance_values in expected_value.items(): + for variable_name, value in instance_values.items(): + entity_index = population.get_index(instance_id) + self.check_variable(variable_name, value, self.test.get('period'), entity_index) + else: + raise VariableNotFound(key, tax_benefit_system) + + def check_variable(self, variable_name, expected_value, period, entity_index = None): + if self.should_ignore_variable(variable_name): + return + if isinstance(expected_value, dict): + for requested_period, expected_value_at_period in expected_value.items(): + self.check_variable(variable_name, expected_value_at_period, requested_period, entity_index) + return + actual_value = self.simulation.calculate(variable_name, period) + if entity_index is not None: + actual_value = actual_value[entity_index] + return assert_near( + actual_value, + expected_value, + absolute_error_margin = self.test.get('absolute_error_margin'), + message = f"In test '{self.test.get('name')}', in file '{self.fspath}', {variable_name}@{period}: ", + relative_error_margin = self.test.get('relative_error_margin'), + ) + + def should_ignore_variable(self, variable_name): + only_variables = self.options.get('only_variables') + ignore_variables = self.options.get('ignore_variables') + variable_ignored = ignore_variables is not None and variable_name in ignore_variables + variable_not_tested = only_variables is not None and variable_name not in only_variables + + return variable_ignored or variable_not_tested - if not isinstance(tests, list): - tests = [tests] - tests = filter(lambda test: test, tests) # Remove empty tests - for test in tests: - yield _parse_test(tax_benefit_system, test, options, yaml_path) +class OpenFiscaPlugin(object): + + def __init__(self, tax_benefit_system, options): + self.tax_benefit_system = tax_benefit_system + self.options = options + + def pytest_collect_file(self, parent, path): + if path.ext == ".yaml": + return YamlFile(path, parent, self.tax_benefit_system, self.options) def _get_tax_benefit_system(baseline, reforms, extensions): @@ -210,89 +242,3 @@ def _get_tax_benefit_system(baseline, reforms, extensions): _tax_benefit_system_cache[key] = current_tax_benefit_system return current_tax_benefit_system - - -def _parse_test(tax_benefit_system, test, options, yaml_path): - name = test.get('name', '') - if not test.get('output'): - raise ValueError("Missing key 'output' in test '{}' in file '{}'".format(name, yaml_path)) - - if not TEST_KEYWORDS.issuperset(test.keys()): - unexpected_keys = set(test.keys()).difference(TEST_KEYWORDS) - raise ValueError("Unexpected keys {} in test '{}' in file '{}'".format(unexpected_keys, name, yaml_path)) - test['file_path'] = yaml_path - - current_tax_benefit_system = _get_tax_benefit_system(tax_benefit_system, test.get('reforms', []), test.get('extensions', [])) - - try: - builder = SimulationBuilder() - input = test.pop('input', {}) - period = test.get('period') - verbose = options.get('verbose') - builder.set_default_period(period) - simulation = builder.build_from_dict(current_tax_benefit_system, input) - simulation.trace = verbose - except SituationParsingError as error: - message = os.linesep.join([ - traceback.format_exc(), - str(error.error), - os.linesep, - "Could not parse situation described in test '{}' in YAML file '{}'. Check the stack trace above for more details.".format(name, yaml_path), - ]) - raise ValueError(message) - except Exception as e: - error_message = os.linesep.join([str(e), '', f"Unexpected error raised while parsing '{test['file_path']}'"]) - raise ValueError(error_message).with_traceback(sys.exc_info()[2]) from e # Keep the stack trace from the root error - return simulation, test - - -def _run_test(simulation, test): - tax_benefit_system = simulation.tax_benefit_system - output = test.get('output') - - if output is None: - return - for key, expected_value in output.items(): - if tax_benefit_system.variables.get(key): # If key is a variable - _check_variable(simulation, key, expected_value, test.get('period'), test) - elif simulation.populations.get(key): # If key is an entity singular - for variable_name, value in expected_value.items(): - _check_variable(simulation, variable_name, value, test.get('period'), test) - else: - population = simulation.get_population(plural = key) - if population is not None: # If key is an entity plural - for instance_id, instance_values in expected_value.items(): - for variable_name, value in instance_values.items(): - entity_index = population.get_index(instance_id) - _check_variable(simulation, variable_name, value, test.get('period'), test, entity_index) - else: - raise VariableNotFound(key, tax_benefit_system) - - -def _should_ignore_variable(variable_name, test): - only_variables = test['options'].get('only_variables') - ignore_variables = test['options'].get('ignore_variables') - variable_ignored = ignore_variables is not None and variable_name in ignore_variables - variable_not_tested = only_variables is not None and variable_name not in only_variables - - return variable_ignored or variable_not_tested - - -def _check_variable(simulation, variable_name, expected_value, period, test, entity_index = None): - if _should_ignore_variable(variable_name, test): - return - if isinstance(expected_value, dict): - for requested_period, expected_value_at_period in expected_value.items(): - _check_variable(simulation, variable_name, expected_value_at_period, requested_period, test, entity_index) - return - actual_value = simulation.calculate(variable_name, period) - if entity_index is not None: - actual_value = actual_value[entity_index] - return assert_near( - actual_value, - expected_value, - absolute_error_margin = test.get('absolute_error_margin'), - message = "In test '{}', in file '{}', {}@{}: ".format( - test.get('name'), test.get('file_path'), variable_name, period), - relative_error_margin = test.get('relative_error_margin'), - ) diff --git a/tests/core/test_yaml.py b/tests/core/test_yaml.py index 7479e83073..1514dfe92b 100644 --- a/tests/core/test_yaml.py +++ b/tests/core/test_yaml.py @@ -2,24 +2,24 @@ import pkg_resources import os -import sys import subprocess from nose.tools import nottest, raises import openfisca_extension_template -from openfisca_core.tools.test_runner import run_tests, generate_tests +from openfisca_core.tools.test_runner import run_tests from .test_countries import tax_benefit_system openfisca_core_dir = pkg_resources.get_distribution('OpenFisca-Core').location yaml_tests_dir = os.path.join(openfisca_core_dir, 'tests', 'core', 'yaml_tests') +EXIT_OK = 0 +EXIT_TESTSFAILED = 1 # Declare that these two functions are not tests to run with nose nottest(run_tests) -nottest(generate_tests) @nottest @@ -29,62 +29,60 @@ def run_yaml_test(path, options = None): if options is None: options = {} - # We are testing tests, and don't want the latter to print anything, so we temporarily deactivate stderr. - sys.stderr = open(os.devnull, 'w') result = run_tests(tax_benefit_system, yaml_path, options) return result def test_success(): - assert run_yaml_test('test_success.yaml') + assert run_yaml_test('test_success.yaml') == EXIT_OK def test_fail(): - assert run_yaml_test('test_failure.yaml') is False + assert run_yaml_test('test_failure.yaml') == EXIT_TESTSFAILED def test_relative_error_margin_success(): - assert run_yaml_test('test_relative_error_margin.yaml') + assert run_yaml_test('test_relative_error_margin.yaml') == EXIT_OK def test_relative_error_margin_fail(): - assert run_yaml_test('failing_test_relative_error_margin.yaml') is False + assert run_yaml_test('failing_test_relative_error_margin.yaml') == EXIT_TESTSFAILED def test_absolute_error_margin_success(): - assert run_yaml_test('test_absolute_error_margin.yaml') + assert run_yaml_test('test_absolute_error_margin.yaml') == EXIT_OK def test_absolute_error_margin_fail(): - assert run_yaml_test('failing_test_absolute_error_margin.yaml') is False + assert run_yaml_test('failing_test_absolute_error_margin.yaml') == EXIT_TESTSFAILED def test_run_tests_from_directory(): dir_path = os.path.join(yaml_tests_dir, 'directory') - assert run_yaml_test(dir_path) + assert run_yaml_test(dir_path) == EXIT_OK def test_with_reform(): - assert run_yaml_test('test_with_reform.yaml') + assert run_yaml_test('test_with_reform.yaml') == EXIT_OK def test_with_extension(): - assert run_yaml_test('test_with_extension.yaml') + assert run_yaml_test('test_with_extension.yaml') == EXIT_OK def test_with_anchors(): - assert run_yaml_test('test_with_anchors.yaml') + assert run_yaml_test('test_with_anchors.yaml') == EXIT_OK def test_run_tests_from_directory_fail(): - assert run_yaml_test(yaml_tests_dir) is False + assert run_yaml_test(yaml_tests_dir) == EXIT_TESTSFAILED def test_name_filter(): assert run_yaml_test( yaml_tests_dir, options = {'name_filter': 'success'} - ) + ) == EXIT_OK def test_shell_script(): diff --git a/tests/core/tools/test_runner/test_yaml_runner.py b/tests/core/tools/test_runner/test_yaml_runner.py index 0cb7db3321..5c9f7a4462 100644 --- a/tests/core/tools/test_runner/test_yaml_runner.py +++ b/tests/core/tools/test_runner/test_yaml_runner.py @@ -1,4 +1,4 @@ -from openfisca_core.tools.test_runner import _run_test, _get_tax_benefit_system +from openfisca_core.tools.test_runner import _get_tax_benefit_system, YamlItem from openfisca_core.errors import VariableNotFound @@ -29,17 +29,24 @@ def __init__(self, baseline): class Simulation: def __init__(self): - self.tax_benefit_system = TaxBenefitSystem() self.populations = {} def get_population(self, plural = None): return None +class TestItem(YamlItem): + def __init__(self, test): + self.test = test + self.simulation = Simulation() + self.tax_benefit_system = TaxBenefitSystem() + + def test_variable_not_found(): test = {"output": {"unknown_variable": 0}} with pytest.raises(VariableNotFound) as excinfo: - _run_test(Simulation(), test) + test_item = TestItem(test) + test_item.check_output() assert excinfo.value.variable_name == "unknown_variable" diff --git a/tests/core/yaml_tests/test_name_filter.yaml b/tests/core/yaml_tests/test_name_filter.yaml index cb15bdc194..9bca0e050d 100644 --- a/tests/core/yaml_tests/test_name_filter.yaml +++ b/tests/core/yaml_tests/test_name_filter.yaml @@ -1,4 +1,4 @@ -- name: "Test that sould be run because the magic word `success is in its title" +- name: "Test that sould be run because the magic word success is in its title" period: 2015-01 input: salary: 2000