From 8fc0d3c879559f8a91ec45c8595a08b239094ac9 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 13:42:26 +0100 Subject: [PATCH 01/40] Draft Entities refactoring --- openfisca_core/entities.py | 218 ++++++++++++------------------------- 1 file changed, 71 insertions(+), 147 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index df7f73b9f1..3a652cbcc0 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import traceback -import warnings import textwrap from os import linesep from typing import Iterable @@ -15,6 +13,15 @@ DIVIDE = 'divide' +def projectable(function): + """ + Decorator to indicate that when called on a projector, the outcome of the function must be projected. + For instance person.household.sum(...) must be projected on person, while it would not make sense for person.household.get_holder. + """ + function.projectable = True + return function + + class Role(object): def __init__(self, description, entity): @@ -30,40 +37,19 @@ def __repr__(self): return "Role({})".format(self.key) -class Entity(object): - """ - Represents an entity (e.g. a person, a household, etc.) on which calculations can be run. - """ - key = None - plural = None - label = None - doc = "" - is_person = False - - def __init__(self, simulation): +class Population(object): + def __init__(self, entity, simulation): + self.entity = entity self.simulation = simulation self._holders = {} self.count = 0 self.ids = [] - self.step_size = 0 - def clone(self, new_simulation): - """ - Returns an entity instance with the same structure, but no variable value set. - """ - new = self.__class__(new_simulation) - new_dict = new.__dict__ - - for key, value in self.__dict__.items(): - if key == '_holders': - new_dict[key] = { - name: holder.clone(new) - for name, holder in self._holders.items() - } - elif key != 'simulation': - new_dict[key] = value + def empty_array(self): + return np.zeros(self.count) - return new + def filled_array(self, value, dtype = None): + return np.full(self.count, value, dtype or float) def __getattr__(self, attribute): projector = get_projector_from_shortcut(self, attribute) @@ -71,19 +57,11 @@ def __getattr__(self, attribute): raise AttributeError("Entity {} has no attribute {}".format(self.key, attribute)) return projector - @classmethod - def to_json(cls): - return { - 'isPersonsEntity': cls.is_person, - 'key': cls.key, - 'label': cls.label, - 'plural': cls.plural, - 'doc': cls.doc, - 'roles': cls.roles_description, - } - # Calculations + def get_variable(self, variable_name, check_existence = False): + return self.simulation.tax_benefit_system.get_variable(variable_name, check_existence) + def check_variable_defined_for_entity(self, variable_name): variable_entity = self.get_variable(variable_name, check_existence = True).entity if not isinstance(self, variable_entity): @@ -99,23 +77,7 @@ def check_array_compatible_with_entity(self, array): raise ValueError("Input {} is not a valid value for the entity {} (size = {} != {} = count)".format( array, self.key, array.size, self.count)) - def check_role_validity(self, role): - if role is not None and not type(role) == Role: - raise ValueError("{} is not a valid role".format(role)) - - def check_period_validity(self, variable_name, period): - if period is None: - stack = traceback.extract_stack() - filename, line_number, function_name, line_of_code = stack[-3] - raise ValueError(''' -You requested computation of variable "{}", but you did not specify on which period in "{}:{}": - {} -When you request the computation of a variable within a formula, you must always specify the period as the second parameter. The convention is to call this parameter "period". For example: - computed_salary = person('salary', period). -See more information at . -'''.format(variable_name, filename, line_number, line_of_code).encode('utf-8')) - - def __call__(self, variable_name, period = None, options = None, **parameters): + def __call__(self, variable_name, period, options = None, **parameters): """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -128,8 +90,6 @@ def __call__(self, variable_name, period = None, options = None, **parameters): """ self.check_variable_defined_for_entity(variable_name) - self.check_period_validity(variable_name, period) - if options is None: options = [] @@ -144,17 +104,6 @@ def __call__(self, variable_name, period = None, options = None, **parameters): # Helpers - def empty_array(self): - return np.zeros(self.count) - - def filled_array(self, value, dtype = None): - with warnings.catch_warnings(): # Avoid a non-relevant warning - warnings.simplefilter("ignore") - return np.full(self.count, value, dtype) - - def get_variable(self, variable_name, check_existence = False): - return self.simulation.tax_benefit_system.get_variable(variable_name, check_existence) - def get_holder(self, variable_name): self.check_variable_defined_for_entity(variable_name) holder = self._holders.get(variable_name) @@ -183,24 +132,6 @@ def get_memory_usage(self, variables = None): by_variable = holders_memory_usage ) - -def projectable(function): - """ - Decorator to indicate that when called on a projector, the outcome of the function must be projected. - For instance person.household.sum(...) must be projected on person, while it would not make sense for person.household.get_holder. - """ - function.projectable = True - return function - - -class PersonEntity(Entity): - """ - Represents a person on which calculations are run. - """ - is_person = True - - # Projection person -> person - @projectable def has_role(self, role): """ @@ -279,23 +210,39 @@ def get_rank(self, entity, criteria, condition = True): return np.where(condition, result, -1) -class GroupEntity(Entity): +class Entity(object): """ - Represents an entity composed of several persons with different roles, on which calculations are run. + Represents an entity (e.g. a person, a household, etc.) on which calculations can be run. """ + def __init__(self, key, label, plural, doc): + self.key = key + self.label = label + self.plural = plural + self.doc = textwrap.dedent(doc) + self.is_person = False - roles = None - flattened_roles = None - roles_description = None + @classmethod + def to_json(cls): + return { + 'isPersonsEntity': cls.is_person, + 'key': cls.key, + 'label': cls.label, + 'plural': cls.plural, + 'doc': cls.doc, + 'roles': cls.roles_description, + } + + def check_role_validity(self, role): + if role is not None and not type(role) == Role: + raise ValueError("{} is not a valid role".format(role)) - def __init__(self, simulation, persons = None): - Entity.__init__(self, simulation) + +class GroupPopulation(Population): + def __init__(self): + super.__init__(self) self._members_entity_id = None self._members_role = None self._members_position = None - self.members = persons - self._roles_count = None - self._ordered_members_map = None @property def members_position(self): @@ -339,16 +286,6 @@ def get_role(self, role_name): def members_position(self, members_position): self._members_position = members_position - @property - def ordered_members_map(self): - """ - Mask to group the persons by entity - This function only caches the map value, to see what the map is used for, see value_nth_person method. - """ - if self._ordered_members_map is None: - return np.argsort(self.members_entity_id) - return self._ordered_members_map - # Aggregation persons -> entity @projectable @@ -551,21 +488,25 @@ def project(self, array, role = None): role_condition = self.members.has_role(role) return np.where(role_condition, array[self.members_entity_id], 0) - # Does it really make sense ? Should not we use roles instead of position when projecting on someone in particular ? - # Doesn't seem to be used, maybe should just not introduce - def project_on_first_person(self, array): - self.check_array_compatible_with_entity(array) - entity_position_array = self.members_position - entity_index_array = self.members_entity_id - position_filter = (entity_position_array == 0) - return np.where(position_filter, array[entity_index_array], 0) - @projectable - def share_between_members(self, array, role = None): - self.check_array_compatible_with_entity(array) - self.check_role_validity(role) - nb_persons_per_entity = self.nb_persons(role) - return self.project(array / nb_persons_per_entity, role = role) +class GroupEntity(Entity): + """ + Represents an entity composed of several persons with different roles, on which calculations are run. + """ + def __init__(self, key, label, plural, doc, roles): + super.__init__(self, key, label, plural, doc) + self.roles_description = roles + self.roles = [] + for role_description in roles: + role = Role(role_description) + self.roles.append(role) + if role_description.get('subroles'): + role.subroles = [] + for subrole_key in role_description['subroles']: + subrole = Role({'key': subrole_key, 'max': 1}) + role.subroles.append(subrole) + role.max = len(role.subroles) + self.flattened_roles = sum([role2.subroles or [role2] for role2 in self.roles], []) class Projector(object): @@ -638,30 +579,6 @@ def transform(self, result): return self.target_entity.value_from_person(result, self.role) -def build_entity(key, plural, label, doc = "", roles = None, is_person = False, class_override = None): - entity_class_name = key.title() - attributes = {'key': key, 'plural': plural, 'label': label, 'doc': textwrap.dedent(doc), 'roles_description': roles} - if is_person: - entity_class = type(entity_class_name, (class_override or PersonEntity,), attributes) - elif roles: - entity_class = type(entity_class_name, (class_override or GroupEntity,), attributes) - entity_class.roles = [] - for role_description in roles: - role = Role(role_description, entity_class) - entity_class.roles.append(role) - setattr(entity_class, role.key.upper(), role) - if role_description.get('subroles'): - role.subroles = [] - for subrole_key in role_description['subroles']: - subrole = Role({'key': subrole_key, 'max': 1}, entity_class) - setattr(entity_class, subrole.key.upper(), subrole) - role.subroles.append(subrole) - role.max = len(role.subroles) - entity_class.flattened_roles = sum([role2.subroles or [role2] for role2 in entity_class.roles], []) - - return entity_class - - def get_projector_from_shortcut(entity, shortcut, parent = None): if entity.is_person: if shortcut in entity.simulation.entities: @@ -673,3 +590,10 @@ def get_projector_from_shortcut(entity, shortcut, parent = None): role = next((role for role in entity.flattened_roles if (role.max == 1) and (role.key == shortcut)), None) if role: return UniqueRoleToEntityProjector(entity, role, parent) + + +def build_entity(key, plural, label, doc = "", roles = None, is_person = False, class_override = None): + if is_person: + return Entity(key, plural, label, doc) + else: + return GroupEntity(key, plural, label, doc, roles) From 30a282ff5740f4ca3ab7790e3bcf0c43e3a4ce88 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:05:29 +0100 Subject: [PATCH 02/40] Complete separation of Entity and Population --- openfisca_core/entities.py | 97 ++++++++++++++------------- openfisca_core/simulation_builder.py | 27 ++++---- openfisca_core/simulations.py | 23 ++++--- openfisca_core/taxbenefitsystems.py | 15 +++-- openfisca_core/variables.py | 9 +-- openfisca_web_api/handlers.py | 5 +- tests/core/test_simulation_builder.py | 16 ++--- 7 files changed, 101 insertions(+), 91 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 3a652cbcc0..06c5c09ea9 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -38,9 +38,9 @@ def __repr__(self): class Population(object): - def __init__(self, entity, simulation): + def __init__(self, entity): + self.simulation = None self.entity = entity - self.simulation = simulation self._holders = {} self.count = 0 self.ids = [] @@ -51,27 +51,14 @@ def empty_array(self): def filled_array(self, value, dtype = None): return np.full(self.count, value, dtype or float) - def __getattr__(self, attribute): - projector = get_projector_from_shortcut(self, attribute) - if not projector: - raise AttributeError("Entity {} has no attribute {}".format(self.key, attribute)) - return projector + # def __getattr__(self, attribute): + # projector = get_projector_from_shortcut(self, attribute) + # if not projector: + # raise AttributeError("Entity {} has no attribute {}".format(self.key, attribute)) + # return projector # Calculations - def get_variable(self, variable_name, check_existence = False): - return self.simulation.tax_benefit_system.get_variable(variable_name, check_existence) - - def check_variable_defined_for_entity(self, variable_name): - variable_entity = self.get_variable(variable_name, check_existence = True).entity - if not isinstance(self, variable_entity): - message = linesep.join([ - "You tried to compute the variable '{0}' for the entity '{1}';".format(variable_name, self.plural), - "however the variable '{0}' is defined for '{1}'.".format(variable_name, variable_entity.plural), - "Learn more about entities in our documentation:", - "."]) - raise ValueError(message) - def check_array_compatible_with_entity(self, array): if not self.count == array.size: raise ValueError("Input {} is not a valid value for the entity {} (size = {} != {} = count)".format( @@ -88,7 +75,7 @@ def __call__(self, variable_name, period, options = None, **parameters): :returns: A numpy array containing the result of the calculation """ - self.check_variable_defined_for_entity(variable_name) + self.entity.check_variable_defined_for_entity(variable_name) if options is None: options = [] @@ -105,11 +92,11 @@ def __call__(self, variable_name, period, options = None, **parameters): # Helpers def get_holder(self, variable_name): - self.check_variable_defined_for_entity(variable_name) + self.entity.check_variable_defined_for_entity(variable_name) holder = self._holders.get(variable_name) if holder: return holder - variable = self.get_variable(variable_name) + variable = self.entity.get_variable(variable_name) self._holders[variable_name] = holder = Holder( entity = self, variable = variable, @@ -142,7 +129,7 @@ def has_role(self, role): >>> person.has_role(Household.CHILD) >>> array([False]) """ - self.check_role_validity(role) + self.entity.check_role_validity(role) group_entity = self.simulation.get_entity(role.entity_class) if role.subroles: return np.logical_or.reduce([group_entity.members_role == subrole for subrole in role.subroles]) @@ -152,7 +139,7 @@ def has_role(self, role): @projectable def value_from_partner(self, array, entity, role): self.check_array_compatible_with_entity(array) - self.check_role_validity(role) + self.entity.check_role_validity(role) if not role.subroles or not len(role.subroles) == 2: raise Exception('Projection to partner is only implemented for roles having exactly two subroles.') @@ -214,35 +201,52 @@ class Entity(object): """ Represents an entity (e.g. a person, a household, etc.) on which calculations can be run. """ - def __init__(self, key, label, plural, doc): + def __init__(self, key, plural, label, doc): self.key = key self.label = label self.plural = plural self.doc = textwrap.dedent(doc) - self.is_person = False + self.is_person = True + self.tax_benefit_system = None - @classmethod - def to_json(cls): + def set_tax_benefit_system(self, tax_benefit_system): + self.tax_benefit_system = tax_benefit_system + + def to_json(self): return { - 'isPersonsEntity': cls.is_person, - 'key': cls.key, - 'label': cls.label, - 'plural': cls.plural, - 'doc': cls.doc, - 'roles': cls.roles_description, + 'isPersonsEntity': self.is_person, + 'key': self.key, + 'label': self.label, + 'plural': self.plural, + 'doc': self.doc, + 'roles': self.roles_description, } def check_role_validity(self, role): if role is not None and not type(role) == Role: raise ValueError("{} is not a valid role".format(role)) + def get_variable(self, variable_name, check_existence = False): + return self.tax_benefit_system.get_variable(variable_name, check_existence) + + def check_variable_defined_for_entity(self, variable_name): + variable_entity = self.get_variable(variable_name, check_existence = True).entity + if variable_entity is not self: + message = linesep.join([ + "You tried to compute the variable '{0}' for the entity '{1}';".format(variable_name, self.plural), + "however the variable '{0}' is defined for '{1}'.".format(variable_name, variable_entity.plural), + "Learn more about entities in our documentation:", + "."]) + raise ValueError(message) + class GroupPopulation(Population): - def __init__(self): - super.__init__(self) + def __init__(self, entity, members): + super().__init__(entity) self._members_entity_id = None self._members_role = None self._members_position = None + self.members = members @property def members_position(self): @@ -303,7 +307,7 @@ def sum(self, array, role = None): >>> household.sum(salaries) >>> array([3500]) """ - self.check_role_validity(role) + self.entity.check_role_validity(role) self.members.check_array_compatible_with_entity(array) if role is not None: role_filter = self.members.has_role(role) @@ -335,7 +339,7 @@ def any(self, array, role = None): @projectable def reduce(self, array, reducer, neutral_element, role = None): self.members.check_array_compatible_with_entity(array) - self.check_role_validity(role) + self.entity.check_role_validity(role) position_in_entity = self.members_position role_filter = self.members.has_role(role) if role is not None else True filtered_array = np.where(role_filter, array, neutral_element) @@ -433,7 +437,7 @@ def value_from_person(self, array, role, default = 0): The result is a vector which dimension is the number of entities """ - self.check_role_validity(role) + self.entity.check_role_validity(role) if role.max != 1: raise Exception( 'You can only use value_from_person with a role that is unique in {}. Role {} is not unique.' @@ -481,7 +485,7 @@ def value_from_first_person(self, array): def project(self, array, role = None): self.check_array_compatible_with_entity(array) - self.check_role_validity(role) + self.entity.check_role_validity(role) if role is None: return array[self.members_entity_id] else: @@ -493,20 +497,23 @@ class GroupEntity(Entity): """ Represents an entity composed of several persons with different roles, on which calculations are run. """ - def __init__(self, key, label, plural, doc, roles): - super.__init__(self, key, label, plural, doc) + def __init__(self, key, plural, label, doc, roles): + super().__init__(key, plural, label, doc) self.roles_description = roles self.roles = [] for role_description in roles: - role = Role(role_description) + role = Role(role_description, self) + setattr(self, role.key.upper(), role) self.roles.append(role) if role_description.get('subroles'): role.subroles = [] for subrole_key in role_description['subroles']: - subrole = Role({'key': subrole_key, 'max': 1}) + subrole = Role({'key': subrole_key, 'max': 1}, self) + setattr(self, subrole.key.upper(), subrole) role.subroles.append(subrole) role.max = len(role.subroles) self.flattened_roles = sum([role2.subroles or [role2] for role2 in self.roles], []) + self.is_person = False class Projector(object): diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index f6ddf8b35c..c3b06c9456 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -97,15 +97,15 @@ def build_from_entities(self, tax_benefit_system, input_dict): raise SituationParsingError([tax_benefit_system.person_entity.plural], 'No {0} found. At least one {0} must be defined to run a simulation.'.format(tax_benefit_system.person_entity.key)) - persons_ids = self.add_person_entity(simulation.persons, persons_json) + persons_ids = self.add_person_entity(simulation.persons.entity, persons_json) for entity_class in tax_benefit_system.group_entities: entity = simulation.entities[entity_class.key] instances_json = input_dict.get(entity_class.plural) if instances_json is not None: - self.add_group_entity(simulation.persons.plural, persons_ids, entity, instances_json) + self.add_group_entity(self.persons_plural, persons_ids, entity_class, instances_json) else: - self.add_default_group_entity(persons_ids, entity) + self.add_default_group_entity(persons_ids, entity_class) if axes: self.axes = axes @@ -174,7 +174,7 @@ def declare_person_entity(self, person_singular, persons_ids: Iterable): person_instance.ids = np.array(list(persons_ids)) person_instance.count = len(person_instance.ids) - self.persons_plural = person_instance.plural + self.persons_plural = person_instance.entity.plural def declare_entity(self, entity_singular, entity_ids: Iterable): entity_instance = self.entities_instances[entity_singular] @@ -377,18 +377,19 @@ def add_variable_value(self, entity, variable, instance_index, instance_id, peri self.input_buffer[variable.name][str(period(period_str))] = array - def finalize_variables_init(self, entity): + def finalize_variables_init(self, population): # Due to set_input mechanism, we must bufferize all inputs, then actually set them, # so that the months are set first and the years last. - if entity.plural in self.entity_counts: - entity.count = self.get_count(entity.plural) - entity.ids = self.get_ids(entity.plural) - if entity.plural in self.memberships: - entity.members_entity_id = np.array(self.get_memberships(entity.plural)) - entity.members_role = np.array(self.get_roles(entity.plural)) + plural_key = population.entity.plural + if plural_key in self.entity_counts: + population.count = self.get_count(plural_key) + population.ids = self.get_ids(plural_key) + if plural_key in self.memberships: + population.members_entity_id = np.array(self.get_memberships(plural_key)) + population.members_role = np.array(self.get_roles(plural_key)) for variable_name in self.input_buffer.keys(): try: - holder = entity.get_holder(variable_name) + holder = population.get_holder(variable_name) except ValueError: # Wrong entity, we can just ignore that continue buffer = self.input_buffer[variable_name] @@ -399,7 +400,7 @@ def finalize_variables_init(self, entity): values = buffer[str(period_value)] # Hack to replicate the values in the persons entity # when we have an axis along a group entity but not persons - array = np.tile(values, entity.count // len(values)) + array = np.tile(values, population.count // len(values)) variable = holder.variable # TODO - this duplicates the check in Simulation.set_input, but # fixing that requires improving Simulation's handling of entities diff --git a/openfisca_core/simulations.py b/openfisca_core/simulations.py index 6c894f04f7..c9e7265d94 100644 --- a/openfisca_core/simulations.py +++ b/openfisca_core/simulations.py @@ -86,9 +86,9 @@ def link_to_entities_instances(self): entity_instance.simulation = self def create_shortcuts(self): - for _key, entity_instance in self.entities.items(): + for _key, population in self.entities.items(): # create shortcut simulation.person and simulation.household (for instance) - setattr(self, entity_instance.key, entity_instance) + setattr(self, population.entity.key, population) @property def data_storage_dir(self): @@ -458,13 +458,20 @@ def set_input(self, variable_name, period, value): def get_variable_entity(self, variable_name): variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True) - return self.get_entity(variable.entity) + return self.entities[variable.entity.key] + + def get_population(self, plural = None): + return next((population for population in self.entities.values() if population.entity.plural == plural), None) + + def get_entity(self, plural = None): + return self.get_population(plural).entity + + def get_index(self, plural, id): + population = self.get_population(plural) + return population.ids.index(id) - def get_entity(self, entity_type = None, plural = None): - if entity_type: - return self.entities[entity_type.key] - if plural: - return next((entity for entity in self.entities.values() if entity.plural == plural), None) + def describe_entities(self): + return {population.entity.plural: population.ids for population in self.entities.values()} def clone(self, debug = False, trace = False): """ diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index 202a80a2e9..db975d3edd 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -13,7 +13,7 @@ import traceback from openfisca_core import periods -from openfisca_core.entities import Entity +from openfisca_core.entities import Entity, Population, GroupPopulation from openfisca_core.parameters import ParameterNode from openfisca_core.variables import Variable, get_neutralized_variable from openfisca_core.errors import VariableNotFound @@ -65,6 +65,8 @@ def __init__(self, entities): raise Exception("A tax and benefit sytem must have at least an entity.") self.person_entity = [entity for entity in entities if entity.is_person][0] self.group_entities = [entity for entity in entities if not entity.is_person] + for entity in entities: + entity.set_tax_benefit_system(self) @property def base_tax_benefit_system(self): @@ -77,13 +79,14 @@ def base_tax_benefit_system(self): return base_tax_benefit_system def instantiate_entities(self): - person_instance = self.person_entity(None) - entities_instances: Dict[Entity.key, Entity] = {person_instance.key: person_instance} + person = self.person_entity + members = Population(person) + entities: Dict[Entity.key, Entity] = {person.key: members} - for entity_class in self.group_entities: - entities_instances[entity_class.key] = entity_class(None, person_instance) + for entity in self.group_entities: + entities[entity.key] = GroupPopulation(entity, members) - return entities_instances + return entities # Deprecated method of constructing simulations, to be phased out in favor of SimulationBuilder def new_scenario(self): diff --git a/openfisca_core/variables.py b/openfisca_core/variables.py index 14689fbe64..ed9695b00f 100644 --- a/openfisca_core/variables.py +++ b/openfisca_core/variables.py @@ -9,7 +9,6 @@ from sortedcontainers.sorteddict import SortedDict from datetime import date -from openfisca_core import entities from openfisca_core import periods from openfisca_core.indexed_enums import Enum, EnumArray, ENUM_ARRAY_DTYPE from openfisca_core.periods import DAY, MONTH, YEAR, ETERNITY @@ -211,9 +210,11 @@ def set(self, attributes, attribute_name, required = False, allowed_values = Non return value def set_entity(self, entity): - if not isinstance(entity, type) or not issubclass(entity, entities.Entity): - raise ValueError("Invalid value '{}' for attribute 'entity' in variable '{}'. Must be a subclass of Entity." - .format(entity, self.name).encode('utf-8')) + # TODO - isinstance() won't work due to (ab)use of load_module to load tax_benefit_system + # Just trust the input in the meantime + # if not isinstance(entity, entities.Entity): + # raise ValueError("Invalid value '{}' for attribute 'entity' in variable '{}'. Must be an instance of Entity." + # .format(entity, self.name).encode('utf-8')) return entity def set_possible_values(self, possible_values): diff --git a/openfisca_web_api/handlers.py b/openfisca_web_api/handlers.py index 0b2223ba11..b65e3b37b7 100644 --- a/openfisca_web_api/handlers.py +++ b/openfisca_web_api/handlers.py @@ -20,8 +20,7 @@ def calculate(tax_benefit_system, input_data): entity_plural, entity_id, variable_name, period = path.split('/') variable = tax_benefit_system.get_variable(variable_name) result = simulation.calculate(variable_name, period) - entity = simulation.get_entity(plural = entity_plural) - entity_index = entity.ids.index(entity_id) + entity_index = simulation.get_index(entity_plural, entity_id) if variable.value_type == Enum: entity_result = result.decode()[entity_index].name @@ -60,6 +59,6 @@ def trace(tax_benefit_system, input_data): return { "trace": trace, - "entitiesDescription": {entity.plural: entity.ids for entity in simulation.entities.values()}, + "entitiesDescription": simulation.describe_entities(), "requestedCalculations": list(simulation.tracer.requested_calculations) } diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index a9195527b0..19d38de0fc 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -10,7 +10,7 @@ from openfisca_core.simulation_builder import SimulationBuilder, Simulation from openfisca_core.tools import assert_near from openfisca_core.tools.test_runner import yaml -from openfisca_core.entities import PersonEntity, GroupEntity, build_entity +from openfisca_core.entities import Entity, GroupEntity from openfisca_core.variables import Variable from openfisca_country_template.entities import Household from openfisca_country_template.situation_examples import couple @@ -85,11 +85,7 @@ def __init__(self): @fixture def persons(): - class TestPersonEntity(PersonEntity): - def __init__(self): - super().__init__(None) - self.plural = "persons" - + class TestPersonEntity(Entity): def get_variable(self, variable_name): result = TestVariable(TestPersonEntity) result.name = variable_name @@ -98,15 +94,12 @@ def get_variable(self, variable_name): def check_variable_defined_for_entity(self, variable_name): return True - return TestPersonEntity() + return TestPersonEntity("person", "persons", "", "") @fixture def group_entity(): class Household(GroupEntity): - def __init__(self): - super().__init__(None) - def get_variable(self, variable_name): result = TestVariable(Household) result.name = variable_name @@ -124,8 +117,7 @@ def check_variable_defined_for_entity(self, variable_name): 'plural': 'children' }] - entity_class = build_entity("household", "households", "", doc = "", roles = roles, is_person = False, class_override = Household) - return entity_class() + return Household("household", "households", "", "", roles) def test_build_default_simulation(simulation_builder): From 9babceea36e2568f17634cb19d75b6feb377b074 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:15:35 +0100 Subject: [PATCH 03/40] Propagate renaming of 'entity' to 'population' --- .../scripts/simulation_generator.py | 4 ++-- openfisca_core/simulation_builder.py | 16 +++++++-------- openfisca_core/simulations.py | 20 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/openfisca_core/scripts/simulation_generator.py b/openfisca_core/scripts/simulation_generator.py index 9ce9a309a8..f2c9eca8fa 100644 --- a/openfisca_core/scripts/simulation_generator.py +++ b/openfisca_core/scripts/simulation_generator.py @@ -61,6 +61,6 @@ def randomly_init_variable(simulation, variable_name, period, max_value, conditi if condition is None: condition = True variable = simulation.tax_benefit_system.get_variable(variable_name) - entity = simulation.get_variable_entity(variable_name) - value = (np.random.rand(entity.count) * max_value * condition).astype(variable.dtype) + population = simulation.get_variable_population(variable_name) + value = (np.random.rand(population.count) * max_value * condition).astype(variable.dtype) simulation.set_input(variable_name, period, value) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index c3b06c9456..19dd0cd633 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -70,9 +70,9 @@ def build_from_entities(self, tax_benefit_system, input_dict): simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) - # Register variables so get_variable_entity can find them + # Register variables so get_variable_population can find them for (variable_name, _variable) in tax_benefit_system.variables.items(): - self.register_variable(variable_name, simulation.get_variable_entity(variable_name)) + self.register_variable(variable_name, simulation.get_variable_population(variable_name)) check_type(input_dict, dict, ['error']) axes = input_dict.pop('axes', None) @@ -159,11 +159,11 @@ def build_default_simulation(self, tax_benefit_system, count = 1): """ simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) - for entity in simulation.entities.values(): - entity.count = count - entity.ids = np.array(range(count)) - if not entity.is_person: - entity.members_entity_id = entity.ids # Each person is its own group entity + for population in simulation.entities.values(): + population.count = count + population.ids = np.array(range(count)) + if not population.entity.is_person: + population.members_entity_id = population.ids # Each person is its own group entity return simulation def create_entities(self, tax_benefit_system): @@ -527,7 +527,7 @@ def expand_axes(self): self.input_buffer[axis_name][str(axis_period)] = array def get_variable_entity(self, variable_name): - return self.variable_entities[variable_name] + return self.variable_entities[variable_name].entity def register_variable(self, variable_name, entity): self.variable_entities[variable_name] = entity diff --git a/openfisca_core/simulations.py b/openfisca_core/simulations.py index c9e7265d94..e0586b37f8 100644 --- a/openfisca_core/simulations.py +++ b/openfisca_core/simulations.py @@ -111,8 +111,8 @@ def calculate(self, variable_name, period, **parameters): :returns: A numpy array containing the result of the calculation """ - entity = self.get_variable_entity(variable_name) - holder = entity.get_holder(variable_name) + population = self.get_variable_population(variable_name) + holder = population.get_holder(variable_name) variable = self.tax_benefit_system.get_variable(variable_name) if period is not None and not isinstance(period, periods.Period): @@ -135,7 +135,7 @@ def calculate(self, variable_name, period, **parameters): # First, try to run a formula try: self._check_for_cycle(variable, period) - array = self._run_formula(variable, entity, period) + array = self._run_formula(variable, population, period) # If no result, use the default value and cache it if array is None: @@ -230,9 +230,9 @@ def trace_parameters_at_instant(self, formula_period): self.tracer ) - def _run_formula(self, variable, entity, period): + def _run_formula(self, variable, population, period): """ - Find the ``variable`` formula for the given ``period`` if it exists, and apply it to ``entity``. + Find the ``variable`` formula for the given ``period`` if it exists, and apply it to ``population``. """ formula = variable.get_formula(period) @@ -245,11 +245,11 @@ def _run_formula(self, variable, entity, period): parameters_at = self.tax_benefit_system.get_parameters_at_instant if formula.__code__.co_argcount == 2: - array = formula(entity, period) + array = formula(population, period) else: - array = formula(entity, period, parameters_at) + array = formula(population, period, parameters_at) - self._check_formula_result(array, variable, entity, period) + self._check_formula_result(array, variable, population, period) return array def _check_period_consistency(self, period, variable): @@ -368,7 +368,7 @@ def get_holder(self, variable_name): """ Get the :any:`Holder` associated with the variable ``variable_name`` for the simulation """ - return self.get_variable_entity(variable_name).get_holder(variable_name) + return self.get_variable_population(variable_name).get_holder(variable_name) def get_memory_usage(self, variables = None): """ @@ -456,7 +456,7 @@ def set_input(self, variable_name, period, value): return self.get_holder(variable_name).set_input(period, value) - def get_variable_entity(self, variable_name): + def get_variable_population(self, variable_name): variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True) return self.entities[variable.entity.key] From 6b5ee55f941214548627fba132c5d6fdba4e3d34 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:32:48 +0100 Subject: [PATCH 04/40] Fix mocks --- openfisca_core/taxbenefitsystems.py | 7 ++++--- tests/core/test_simulation_builder.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index db975d3edd..9eda419c16 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -303,10 +303,11 @@ def get_variable(self, variable_name, check_existence = False): :param variable_name: Name of the requested variable. :param check_existence: If True, raise an error if the requested variable does not exist. """ - variables = self.variables.get(variable_name) - if not variables and check_existence: + variables = self.variables + found = variables.get(variable_name) + if not found and check_existence: raise VariableNotFound(variable_name, self) - return variables + return found def neutralize_variable(self, variable_name): """ diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 19d38de0fc..a44b27a9b8 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -87,7 +87,7 @@ def __init__(self): def persons(): class TestPersonEntity(Entity): def get_variable(self, variable_name): - result = TestVariable(TestPersonEntity) + result = TestVariable(self) result.name = variable_name return result @@ -101,7 +101,7 @@ def check_variable_defined_for_entity(self, variable_name): def group_entity(): class Household(GroupEntity): def get_variable(self, variable_name): - result = TestVariable(Household) + result = TestVariable(self) result.name = variable_name return result From d788905926f601de3bf17c8341970f20f2091854 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:36:28 +0100 Subject: [PATCH 05/40] Fix entity/population confusion --- openfisca_core/simulation_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index 19dd0cd633..1a12f8a293 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -70,9 +70,9 @@ def build_from_entities(self, tax_benefit_system, input_dict): simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) - # Register variables so get_variable_population can find them + # Register variables so get_variable_entity can find them for (variable_name, _variable) in tax_benefit_system.variables.items(): - self.register_variable(variable_name, simulation.get_variable_population(variable_name)) + self.register_variable(variable_name, simulation.get_variable_population(variable_name).entity) check_type(input_dict, dict, ['error']) axes = input_dict.pop('axes', None) @@ -527,7 +527,7 @@ def expand_axes(self): self.input_buffer[axis_name][str(axis_period)] = array def get_variable_entity(self, variable_name): - return self.variable_entities[variable_name].entity + return self.variable_entities[variable_name] def register_variable(self, variable_name, entity): self.variable_entities[variable_name] = entity From 16d92bd4b98420841aa1937d769f15d260fd17c3 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:39:50 +0100 Subject: [PATCH 06/40] Reference flattened_roles of entity --- openfisca_core/entities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 06c5c09ea9..14c3a0ebbd 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -274,7 +274,7 @@ def members_entity_id(self, members_entity_id): @property def members_role(self): if self._members_role is None: - default_role = self.flattened_roles[0] + default_role = self.entity.flattened_roles[0] self._members_role = np.repeat(default_role, len(self.members_entity_id)) return self._members_role @@ -284,7 +284,7 @@ def members_role(self, members_role: Iterable[Role]): self._members_role = np.array(list(members_role)) def get_role(self, role_name): - return next((role for role in self.flattened_roles if role.key == role_name), None) + return next((role for role in self.entity.flattened_roles if role.key == role_name), None) @members_position.setter def members_position(self, members_position): From 3eec517be20b24fe8c7bea002d45f21fd099c463 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:48:14 +0100 Subject: [PATCH 07/40] Fix Builder tests --- tests/core/test_simulation_builder.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index a44b27a9b8..3393ad5fa2 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -10,7 +10,7 @@ from openfisca_core.simulation_builder import SimulationBuilder, Simulation from openfisca_core.tools import assert_near from openfisca_core.tools.test_runner import yaml -from openfisca_core.entities import Entity, GroupEntity +from openfisca_core.entities import Entity, GroupEntity, Population, GroupPopulation from openfisca_core.variables import Variable from openfisca_country_template.entities import Household from openfisca_country_template.situation_examples import couple @@ -259,22 +259,24 @@ def test_add_unknown_enum_variable_value(simulation_builder, persons, enum_varia def test_finalize_person_entity(simulation_builder, persons): persons_json = {'Alicia': {'salary': {'2018-11': 3000}}, 'Javier': {}} simulation_builder.add_person_entity(persons, persons_json) - simulation_builder.finalize_variables_init(persons) - assert_near(persons.get_holder('salary').get_array('2018-11'), [3000, 0]) - assert persons.count == 2 - assert persons.ids == ['Alicia', 'Javier'] + population = Population(persons) + simulation_builder.finalize_variables_init(population) + assert_near(population.get_holder('salary').get_array('2018-11'), [3000, 0]) + assert population.count == 2 + assert population.ids == ['Alicia', 'Javier'] def test_canonicalize_period_keys(simulation_builder, persons): persons_json = {'Alicia': {'salary': {'year:2018-01': 100}}} simulation_builder.add_person_entity(persons, persons_json) - simulation_builder.finalize_variables_init(persons) - assert_near(persons.get_holder('salary').get_array('2018-12'), [100]) + population = Population(persons) + simulation_builder.finalize_variables_init(population) + assert_near(population.get_holder('salary').get_array('2018-12'), [100]) def test_finalize_group_entity(simulation_builder): simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) - simulation_builder.add_group_entity('persons', ['Alicia', 'Javier', 'Sarah', 'Tom'], simulation.household, { + simulation_builder.add_group_entity('persons', ['Alicia', 'Javier', 'Sarah', 'Tom'], simulation.household.entity, { 'Household_1': {'parents': ['Alicia', 'Javier']}, 'Household_2': {'parents': ['Tom'], 'children': ['Sarah']}, }) From e2908ae7fbbf03f009fe045e084ae83a644101b8 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:48:21 +0100 Subject: [PATCH 08/40] Fix has_role --- openfisca_core/entities.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 14c3a0ebbd..ab1072b4c2 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -25,7 +25,7 @@ def projectable(function): class Role(object): def __init__(self, description, entity): - self.entity_class = entity + self.entity = entity self.key = description['key'] self.label = description.get('label') self.plural = description.get('plural') @@ -130,11 +130,11 @@ def has_role(self, role): >>> array([False]) """ self.entity.check_role_validity(role) - group_entity = self.simulation.get_entity(role.entity_class) + group_population = self.simulation.get_population(role.entity.plural) if role.subroles: - return np.logical_or.reduce([group_entity.members_role == subrole for subrole in role.subroles]) + return np.logical_or.reduce([group_population.members_role == subrole for subrole in role.subroles]) else: - return group_entity.members_role == role + return group_population.members_role == role @projectable def value_from_partner(self, array, entity, role): From 878734c3f737845b459f50be01807a83dcfb2584 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:51:59 +0100 Subject: [PATCH 09/40] Fix role access --- tests/core/test_simulation_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 3393ad5fa2..92848f9a12 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -349,7 +349,7 @@ def test_some_person_without_household(simulation_builder): """ simulation = simulation_builder.build_from_dict(tax_benefit_system, yaml.safe_load(input_yaml)) assert simulation.household.count == 2 - parents_in_households = simulation.household.nb_persons(role = simulation.household.PARENT) + parents_in_households = simulation.household.nb_persons(role = Household.PARENT) assert parents_in_households.tolist() == [1, 1] # household member default role is first_parent From 205823c0436c50b3f21f9acf184d7373c2c8a3b4 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:52:10 +0100 Subject: [PATCH 10/40] Fix join_with_persons --- openfisca_core/simulation_builder.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index 1a12f8a293..2ff6e4c474 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -185,11 +185,12 @@ def declare_entity(self, entity_singular, entity_ids: Iterable): def nb_persons(self, entity_singular, role = None): return self.entities_instances[entity_singular].nb_persons(role = role) - def join_with_persons(self, group_instance, persons_group_assignment, roles: Iterable[str]): + def join_with_persons(self, group_population, persons_group_assignment, roles: Iterable[str]): group_sorted_indices = np.unique(persons_group_assignment, return_inverse = True)[1] - group_instance.members_entity_id = np.argsort(group_instance.ids)[group_sorted_indices] + group_population.members_entity_id = np.argsort(group_population.ids)[group_sorted_indices] + flattened_roles = group_population.entity.flattened_roles role_names_array = np.array(roles) - group_instance.members_role = np.select([role_names_array == role.key for role in group_instance.flattened_roles], group_instance.flattened_roles) + group_population.members_role = np.select([role_names_array == role.key for role in flattened_roles], flattened_roles) def build(self, tax_benefit_system): return Simulation(tax_benefit_system, entities_instances = self.entities_instances) From 120d2dbb39dc9760699f946b9a97d2574a45a2f4 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:55:36 +0100 Subject: [PATCH 11/40] Fix role access --- tests/core/test_simulation_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 92848f9a12..24320b3e86 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -378,7 +378,7 @@ def test_nb_persons_no_role(simulation_builder): household_instance = simulation_builder.declare_entity('household', households_ids) simulation_builder.join_with_persons(household_instance, persons_households, ['first_parent'] * 5) - parents_in_households = household_instance.nb_persons(role = household_instance.PARENT) + parents_in_households = household_instance.nb_persons(role = Household.PARENT) assert parents_in_households.tolist() == [1, 3, 1] # household member default role is first_parent @@ -398,7 +398,7 @@ def test_nb_persons_by_role(simulation_builder): persons_households, persons_households_roles ) - parents_in_households = household_instance.nb_persons(role = household_instance.FIRST_PARENT) + parents_in_households = household_instance.nb_persons(role = Household.FIRST_PARENT) assert parents_in_households.tolist() == [0, 1, 1] From adbe5cbd9fab4567d2e49b09b2844398ae9a2aa7 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:55:44 +0100 Subject: [PATCH 12/40] Fix holders --- openfisca_core/holders.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/openfisca_core/holders.py b/openfisca_core/holders.py index 42381dea9e..0e044ef9e1 100644 --- a/openfisca_core/holders.py +++ b/openfisca_core/holders.py @@ -24,10 +24,10 @@ class Holder(object): A holder keeps tracks of a variable values after they have been calculated, or set as an input. """ - def __init__(self, variable, entity): - self.entity = entity + def __init__(self, variable, population): + self.population = population self.variable = variable - self.simulation = entity.simulation + self.simulation = population.simulation self._memory_storage = InMemoryStorage(is_eternal = (self.variable.definition_period == ETERNITY)) # By default, do not activate on-disk storage, or variable dropping @@ -41,7 +41,7 @@ def __init__(self, variable, entity): if self.variable.name in self.simulation.memory_config.variables_to_drop: self._do_not_store = True - def clone(self, entity): + def clone(self, population): """ Copy the holder just enough to be able to run a new simulation without modifying the original simulation. """ @@ -49,11 +49,11 @@ def clone(self, entity): new_dict = new.__dict__ for key, value in self.__dict__.items(): - if key not in ('entity', 'formula', 'simulation'): + if key not in ('population', 'formula', 'simulation'): new_dict[key] = value - new_dict['entity'] = entity - new_dict['simulation'] = entity.simulation + new_dict['population'] = population + new_dict['simulation'] = population.simulation return new @@ -117,7 +117,7 @@ def get_memory_usage(self): """ usage = dict( - nb_cells_by_array = self.entity.count, + nb_cells_by_array = self.population.count, dtype = self.variable.dtype, ) @@ -192,10 +192,10 @@ def _to_array(self, value): if value.ndim == 0: # 0-dim arrays are casted to scalar when they interact with float. We don't want that. value = value.reshape(1) - if len(value) != self.entity.count: + if len(value) != self.population.count: raise ValueError( 'Unable to set value "{}" for variable "{}", as its length is {} while there are {} {} in the simulation.' - .format(value, self.variable.name, len(value), self.entity.count, self.entity.plural)) + .format(value, self.variable.name, len(value), self.population.count, self.population.entity.plural)) if self.variable.value_type == Enum: value = self.variable.possible_values.encode(value) if value.dtype != self.variable.dtype: @@ -255,7 +255,7 @@ def default_array(self): Return a new array of the appropriate length for the entity, filled with the variable default values. """ - return self.variable.default_array(self.entity.count) + return self.variable.default_array(self.population.count) def set_input_dispatch_by_period(holder, period, array): From 63ff5dea69b63b039d16cefe41cb1c55fb981f9a Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 19:58:09 +0100 Subject: [PATCH 13/40] Fix Holder instantiation --- openfisca_core/entities.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index ab1072b4c2..3b805c0c0e 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -97,10 +97,7 @@ def get_holder(self, variable_name): if holder: return holder variable = self.entity.get_variable(variable_name) - self._holders[variable_name] = holder = Holder( - entity = self, - variable = variable, - ) + self._holders[variable_name] = holder = Holder(variable, self) return holder def get_memory_usage(self, variables = None): From e378caf63d7713beee37b12ea96d85355d79cd85 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 20:13:39 +0100 Subject: [PATCH 14/40] Fix shortcuts --- openfisca_core/entities.py | 35 +++++++++++++++++++++++------------ tests/core/test_entities.py | 34 +--------------------------------- 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 3b805c0c0e..95ef9d1572 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -51,11 +51,11 @@ def empty_array(self): def filled_array(self, value, dtype = None): return np.full(self.count, value, dtype or float) - # def __getattr__(self, attribute): - # projector = get_projector_from_shortcut(self, attribute) - # if not projector: - # raise AttributeError("Entity {} has no attribute {}".format(self.key, attribute)) - # return projector + def __getattr__(self, attribute): + projector = get_projector_from_shortcut(self, attribute) + if not projector: + raise AttributeError("Entity {} has no attribute {}".format(self.entity.key, attribute)) + return projector # Calculations @@ -244,6 +244,7 @@ def __init__(self, entity, members): self._members_role = None self._members_position = None self.members = members + self._ordered_members_map = None @property def members_position(self): @@ -280,6 +281,16 @@ def members_role(self, members_role: Iterable[Role]): if members_role is not None: self._members_role = np.array(list(members_role)) + @property + def ordered_members_map(self): + """ + Mask to group the persons by entity + This function only caches the map value, to see what the map is used for, see value_nth_person method. + """ + if self._ordered_members_map is None: + return np.argsort(self.members_entity_id) + return self._ordered_members_map + def get_role(self, role_name): return next((role for role in self.entity.flattened_roles if role.key == role_name), None) @@ -583,17 +594,17 @@ def transform(self, result): return self.target_entity.value_from_person(result, self.role) -def get_projector_from_shortcut(entity, shortcut, parent = None): - if entity.is_person: - if shortcut in entity.simulation.entities: - entity_2 = entity.simulation.entities[shortcut] +def get_projector_from_shortcut(population, shortcut, parent = None): + if population.entity.is_person: + if shortcut in population.simulation.entities: + entity_2 = population.simulation.entities[shortcut] return EntityToPersonProjector(entity_2, parent) else: if shortcut == 'first_person': - return FirstPersonToEntityProjector(entity, parent) - role = next((role for role in entity.flattened_roles if (role.max == 1) and (role.key == shortcut)), None) + return FirstPersonToEntityProjector(population, parent) + role = next((role for role in population.entity.flattened_roles if (role.max == 1) and (role.key == shortcut)), None) if role: - return UniqueRoleToEntityProjector(entity, role, parent) + return UniqueRoleToEntityProjector(population, role, parent) def build_entity(key, plural, label, doc = "", roles = None, is_person = False, class_override = None): diff --git a/tests/core/test_entities.py b/tests/core/test_entities.py index cb3471b951..0702ceaf3b 100644 --- a/tests/core/test_entities.py +++ b/tests/core/test_entities.py @@ -221,35 +221,6 @@ def test_implicit_projection(): assert_near(housing_tax, [20000, 20000, 20000, 20000, 0, 0]) -def test_project_on_first_person(): - test_case = deepcopy(TEST_CASE) - test_case['households']['h1']['housing_tax'] = 20000 - test_case['households']['h2']['housing_tax'] = 5000 - - simulation = new_simulation(test_case, YEAR) - household = simulation.household - - housing_tax = household('housing_tax', YEAR) - projected_housing_tax = household.project_on_first_person(housing_tax) - - assert_near(projected_housing_tax, [20000, 0, 0, 0, 5000, 0]) - - -def test_share_between_members(): - test_case = deepcopy(TEST_CASE) - test_case['households']['h1']['housing_tax'] = 20000 - test_case['households']['h2']['housing_tax'] = 5000 - - simulation = new_simulation(test_case, YEAR) - household = simulation.household - - housing_tax = household('housing_tax', YEAR) - - housing_tax_shared = household.share_between_members(housing_tax, role = PARENT) - - assert_near(housing_tax_shared, [10000, 10000, 0, 0, 5000, 0]) - - def test_sum(): test_case = deepcopy(TEST_CASE) test_case['persons']['ind0']['salary'] = 1000 @@ -502,7 +473,7 @@ def test_unordered_persons(): assert_near(household.first_person('salary', "2016-01"), [0, 3000]) assert_near(household.first_parent('salary', "2016-01"), [1000, 3000]) assert_near(household.second_parent('salary', "2016-01"), [1500, 0]) - assert_near(person.value_from_partner(salary, person.household, household.PARENT), [0, 0, 1000, 0, 0, 1500]) + assert_near(person.value_from_partner(salary, person.household, PARENT), [0, 0, 1000, 0, 0, 1500]) assert_near(household.sum(salary, role = PARENT), [2500, 3000]) assert_near(household.sum(salary, role = CHILD), [20, 500]) @@ -526,6 +497,3 @@ def test_unordered_persons(): assert_near(household.project(accommodation_size), [60, 160, 160, 160, 60, 160]) assert_near(household.project(accommodation_size, role = PARENT), [60, 0, 160, 0, 0, 160]) assert_near(household.project(accommodation_size, role = CHILD), [0, 160, 0, 160, 60, 0]) - assert_near(household.project_on_first_person(accommodation_size), [60, 160, 0, 0, 0, 0]) - assert_near(household.share_between_members(accommodation_size), [30, 40, 40, 40, 30, 40]) - assert_near(household.share_between_members(accommodation_size, role = PARENT), [60, 0, 80, 0, 0, 80]) From b80929e9424f3f413506e1820d9adbfcfb634fbd Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 20:35:37 +0100 Subject: [PATCH 15/40] Handle some identity issues --- openfisca_core/entities.py | 4 +++- openfisca_core/taxbenefitsystems.py | 10 ++++++---- tests/core/test_formulas.py | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 95ef9d1572..e6b9f169d1 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -228,7 +228,9 @@ def get_variable(self, variable_name, check_existence = False): def check_variable_defined_for_entity(self, variable_name): variable_entity = self.get_variable(variable_name, check_existence = True).entity - if variable_entity is not self: + # Should be this: + # if variable_entity is not self: + if variable_entity.key != self.key: message = linesep.join([ "You tried to compute the variable '{0}' for the entity '{1}';".format(variable_name, self.plural), "however the variable '{0}' is defined for '{1}'.".format(variable_name, variable_entity.plural), diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index 9eda419c16..ff93ce3fd1 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -11,6 +11,7 @@ import inspect import pkg_resources import traceback +import copy from openfisca_core import periods from openfisca_core.entities import Entity, Population, GroupPopulation @@ -60,12 +61,13 @@ def __init__(self, entities): self._parameters_at_instant_cache = {} # weakref.WeakValueDictionary() self.variables = {} self.open_api_config = {} - self.entities = entities + # Tax benefit systems are mutable, so entities (which need to know about our variables) can't be shared among them if entities is None or len(entities) == 0: raise Exception("A tax and benefit sytem must have at least an entity.") - self.person_entity = [entity for entity in entities if entity.is_person][0] - self.group_entities = [entity for entity in entities if not entity.is_person] - for entity in entities: + self.entities = [copy.copy(entity) for entity in entities] + self.person_entity = [entity for entity in self.entities if entity.is_person][0] + self.group_entities = [entity for entity in self.entities if not entity.is_person] + for entity in self.entities: entity.set_tax_benefit_system(self) @property diff --git a/tests/core/test_formulas.py b/tests/core/test_formulas.py index 6d0315627d..5da10168a6 100644 --- a/tests/core/test_formulas.py +++ b/tests/core/test_formulas.py @@ -50,8 +50,8 @@ def formula(person, period): # TaxBenefitSystem instance declared after formulas -tax_benefit_system = CountryTaxBenefitSystem() -tax_benefit_system.add_variables(choice, uses_multiplication, uses_switch) +ourtbs = CountryTaxBenefitSystem() +ourtbs.add_variables(choice, uses_multiplication, uses_switch) @fixture @@ -63,7 +63,7 @@ def month(): def simulation(month): builder = SimulationBuilder() builder.default_period = month - simulation = builder.build_from_variables(tax_benefit_system, {'choice': np.random.randint(2, size = 1000) + 1}) + simulation = builder.build_from_variables(ourtbs, {'choice': np.random.randint(2, size = 1000) + 1}) simulation.debug = True return simulation From b61d6033a4701778fe217fe3587ec045c27e3adf Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 20:41:19 +0100 Subject: [PATCH 16/40] Distinguish population and entity --- openfisca_core/simulation_builder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index 2ff6e4c474..2c52619b7c 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -114,14 +114,14 @@ def build_from_entities(self, tax_benefit_system, input_dict): try: self.finalize_variables_init(simulation.persons) except PeriodMismatchError as e: - self.raise_period_mismatch(simulation.persons, persons_json, e) + self.raise_period_mismatch(simulation.persons.entity, persons_json, e) for entity_class in tax_benefit_system.group_entities: try: - entity = simulation.entities[entity_class.key] - self.finalize_variables_init(entity) + population = simulation.entities[entity_class.key] + self.finalize_variables_init(population) except PeriodMismatchError as e: - self.raise_period_mismatch(entity, instances_json, e) + self.raise_period_mismatch(population.entity, instances_json, e) return simulation From 6f08a24f1b40023c0316378e39cc4641e7ccd527 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 20:45:42 +0100 Subject: [PATCH 17/40] Fix spec generation --- openfisca_core/taxbenefitsystems.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index ff93ce3fd1..97d1bdd97e 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -433,7 +433,8 @@ def get_variables(self, entity = None): return { variable_name: variable for variable_name, variable in self.variables.items() - if variable.entity == entity + # TODO - because entities are copied (see constructor) they can't be compared + if variable.entity.key == entity.key } def clone(self): From 2e1f45ede26f8559f2f6b9c5b3a5502e3d775587 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 20:47:23 +0100 Subject: [PATCH 18/40] Remove test for 'no period' now caught at compile time --- tests/core/test_countries.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tests/core/test_countries.py b/tests/core/test_countries.py index 4a36e93512..b8b4eff28e 100644 --- a/tests/core/test_countries.py +++ b/tests/core/test_countries.py @@ -136,25 +136,3 @@ def formula(household, period): return household.empty_array() with raises(VariableNameConflict): tax_benefit_system.add_variable(disposable_income) - - -class income_tax_no_period(Variable): - value_type = float - entity = Person - label = "Salaire net (buggy)" - definition_period = MONTH - - def formula(individu, period): - # salary = individu('salary', period) # correct - salary = individu('salary') # buggy - - return salary * 0.15 - - -def test_no_period(make_isolated_simulation, period): - buggy_tbf = CountryTaxBenefitSystem() - buggy_tbf.add_variable(income_tax_no_period) - - simulation = make_isolated_simulation(buggy_tbf, {'salary': 2000}) - with raises(ValueError): - simulation.calculate('income_tax_no_period', period) From 42346b3de413688cd61fc1350006fa65d74e9139 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 23:12:30 +0100 Subject: [PATCH 19/40] Fix test runner --- openfisca_core/tools/test_runner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 6dee96e8ac..8d8242bd65 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -258,11 +258,11 @@ def _run_test(simulation, test): for variable_name, value in expected_value.items(): _check_variable(simulation, variable_name, value, test.get('period'), test) else: - entity_array = simulation.get_entity(plural = key) - if entity_array is not None: # If key is an entity plural - for entity_id, value_by_entity in expected_value.items(): - for variable_name, value in value_by_entity.items(): - entity_index = entity_array.ids.index(entity_id) + population = simulation.get_population(plural = key) + if population is not None: # If key is an entity plural + for instance_id, instance_values in expected_value.items(): + for variable_name, value in instance_values.items(): + entity_index = simulation.get_index(key, instance_id) _check_variable(simulation, variable_name, value, test.get('period'), test, entity_index) else: raise VariableNotFound(key, tax_benefit_system) From b2dae2241e921b804604792a0f97ac5f10c8c23d Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 23:15:23 +0100 Subject: [PATCH 20/40] Fix mock in runner test --- tests/core/tools/test_runner/test_yaml_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/tools/test_runner/test_yaml_runner.py b/tests/core/tools/test_runner/test_yaml_runner.py index ee4635902a..f4aa22d6dd 100644 --- a/tests/core/tools/test_runner/test_yaml_runner.py +++ b/tests/core/tools/test_runner/test_yaml_runner.py @@ -32,7 +32,7 @@ def __init__(self): self.tax_benefit_system = TaxBenefitSystem() self.entities = {} - def get_entity(self, plural = None): + def get_population(self, plural = None): return None From 7acfbe503f157c8473f9effe2700d0a542a7eabe Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 23:40:47 +0100 Subject: [PATCH 21/40] Fix clone --- openfisca_core/entities.py | 22 +++++++++++++++++++++- openfisca_core/simulations.py | 19 ++++++++++--------- tests/core/test_simulations.py | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index e6b9f169d1..21d61212aa 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -45,6 +45,14 @@ def __init__(self, entity): self.count = 0 self.ids = [] + def clone(self, simulation): + result = Population(self.entity) + result.simulation = simulation + result._holders = {variable: holder.clone(result) for (variable, holder) in self._holders.items()} + result.count = self.count + result.ids = self.ids + return result + def empty_array(self): return np.zeros(self.count) @@ -242,12 +250,24 @@ def check_variable_defined_for_entity(self, variable_name): class GroupPopulation(Population): def __init__(self, entity, members): super().__init__(entity) + self.members = members self._members_entity_id = None self._members_role = None self._members_position = None - self.members = members self._ordered_members_map = None + def clone(self, simulation): + result = GroupPopulation(self.entity, self.members) + result.simulation = simulation + result._holders = {variable: holder.clone(self) for (variable, holder) in self._holders.items()} + result.count = self.count + result.ids = self.ids + result._members_entity_id = self._members_entity_id + result._members_role = self._members_role + result._members_position = self._members_position + result._ordered_members_map = self._ordered_members_map + return result + @property def members_position(self): if self._members_position is None and self.members_entity_id is not None: diff --git a/openfisca_core/simulations.py b/openfisca_core/simulations.py index e0586b37f8..a202dbf91a 100644 --- a/openfisca_core/simulations.py +++ b/openfisca_core/simulations.py @@ -39,7 +39,7 @@ class Simulation(object): def __init__( self, tax_benefit_system, - entities_instances = None + populations ): """ This constructor is reserved for internal use; see :any:`SimulationBuilder`, @@ -49,7 +49,7 @@ def __init__( self.tax_benefit_system = tax_benefit_system assert tax_benefit_system is not None - self.entities = entities_instances + self.entities = populations self.persons = self.entities[tax_benefit_system.person_entity.key] self.link_to_entities_instances() self.create_shortcuts() @@ -464,7 +464,8 @@ def get_population(self, plural = None): return next((population for population in self.entities.values() if population.entity.plural == plural), None) def get_entity(self, plural = None): - return self.get_population(plural).entity + population = self.get_population(plural) + return population and population.entity def get_index(self, plural, id): population = self.get_population(plural) @@ -485,13 +486,13 @@ def clone(self, debug = False, trace = False): new_dict[key] = value new.persons = self.persons.clone(new) - setattr(new, new.persons.key, new.persons) - new.entities = {new.persons.key: new.persons} + setattr(new, new.persons.entity.key, new.persons) + new.entities = {new.persons.entity.key: new.persons} - for entity_class in self.tax_benefit_system.group_entities: - entity = self.entities[entity_class.key].clone(new) - new.entities[entity.key] = entity - setattr(new, entity_class.key, entity) # create shortcut simulation.household (for instance) + for entity in self.tax_benefit_system.group_entities: + population = self.entities[entity.key].clone(new) + new.entities[entity.key] = population + setattr(new, entity.key, population) # create shortcut simulation.household (for instance) new.debug = debug new.trace = trace diff --git a/tests/core/test_simulations.py b/tests/core/test_simulations.py index 16c5f5ae1c..45f20cb8ae 100644 --- a/tests/core/test_simulations.py +++ b/tests/core/test_simulations.py @@ -58,7 +58,7 @@ def test_clone(): assert salary_holder != salary_holder_clone assert salary_holder_clone.simulation == simulation_clone - assert salary_holder_clone.entity == simulation_clone.person + assert salary_holder_clone.population == simulation_clone.persons def test_get_memory_usage(): From 660be260370f3894306d0cfcc91b74b895cbdd45 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sat, 23 Mar 2019 23:46:20 +0100 Subject: [PATCH 22/40] Fix linting --- openfisca_core/simulation_builder.py | 3 +-- tests/core/test_countries.py | 1 - tests/core/test_simulation_builder.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index 2c52619b7c..764c2a50ec 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -100,7 +100,6 @@ def build_from_entities(self, tax_benefit_system, input_dict): persons_ids = self.add_person_entity(simulation.persons.entity, persons_json) for entity_class in tax_benefit_system.group_entities: - entity = simulation.entities[entity_class.key] instances_json = input_dict.get(entity_class.plural) if instances_json is not None: self.add_group_entity(self.persons_plural, persons_ids, entity_class, instances_json) @@ -193,7 +192,7 @@ def join_with_persons(self, group_population, persons_group_assignment, roles: I group_population.members_role = np.select([role_names_array == role.key for role in flattened_roles], flattened_roles) def build(self, tax_benefit_system): - return Simulation(tax_benefit_system, entities_instances = self.entities_instances) + return Simulation(tax_benefit_system, self.entities_instances) def explicit_singular_entities(self, tax_benefit_system, input_dict): """ diff --git a/tests/core/test_countries.py b/tests/core/test_countries.py index b8b4eff28e..307490a195 100644 --- a/tests/core/test_countries.py +++ b/tests/core/test_countries.py @@ -9,7 +9,6 @@ from openfisca_core import periods from openfisca_core.entities import DIVIDE from openfisca_country_template import CountryTaxBenefitSystem -from openfisca_country_template.entities import Person from openfisca_core.tools import assert_near diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 24320b3e86..4f489dd6a3 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -10,7 +10,7 @@ from openfisca_core.simulation_builder import SimulationBuilder, Simulation from openfisca_core.tools import assert_near from openfisca_core.tools.test_runner import yaml -from openfisca_core.entities import Entity, GroupEntity, Population, GroupPopulation +from openfisca_core.entities import Entity, GroupEntity, Population from openfisca_core.variables import Variable from openfisca_country_template.entities import Household from openfisca_country_template.situation_examples import couple From 6b82e3e750326dfc00cbd4276c151dd45582ae04 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 12:22:52 +0100 Subject: [PATCH 23/40] Use population name inside Builder --- openfisca_core/simulation_builder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index 764c2a50ec..c8e3e8b7bf 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -5,7 +5,7 @@ import numpy as np from copy import deepcopy -from openfisca_core.entities import Entity +from openfisca_core.entities import Entity, Population from openfisca_core.variables import Variable from openfisca_core.errors import VariableNotFound, SituationParsingError, PeriodMismatchError @@ -21,7 +21,7 @@ def __init__(self): # JSON input - Memory of known input values. Indexed by variable or axis name. self.input_buffer: Dict[Variable.name, Dict[str(period), np.array]] = {} - self.entities_instances: Dict[Entity.key, Entity] = {} + self.populations: Dict[Entity.key, Population] = {} # JSON input - Number of items of each entity type. Indexed by entities plural names. Should be consistent with ``entity_ids``, including axes. self.entity_counts: Dict[Entity.plural, int] = {} # JSON input - List of items of each entity type. Indexed by entities plural names. Should be consistent with ``entity_counts``. @@ -166,23 +166,23 @@ def build_default_simulation(self, tax_benefit_system, count = 1): return simulation def create_entities(self, tax_benefit_system): - self.entities_instances = tax_benefit_system.instantiate_entities() + self.populations = tax_benefit_system.instantiate_entities() def declare_person_entity(self, person_singular, persons_ids: Iterable): - person_instance = self.entities_instances[person_singular] + person_instance = self.populations[person_singular] person_instance.ids = np.array(list(persons_ids)) person_instance.count = len(person_instance.ids) self.persons_plural = person_instance.entity.plural def declare_entity(self, entity_singular, entity_ids: Iterable): - entity_instance = self.entities_instances[entity_singular] + entity_instance = self.populations[entity_singular] entity_instance.ids = np.array(list(entity_ids)) entity_instance.count = len(entity_instance.ids) return entity_instance def nb_persons(self, entity_singular, role = None): - return self.entities_instances[entity_singular].nb_persons(role = role) + return self.populations[entity_singular].nb_persons(role = role) def join_with_persons(self, group_population, persons_group_assignment, roles: Iterable[str]): group_sorted_indices = np.unique(persons_group_assignment, return_inverse = True)[1] @@ -192,7 +192,7 @@ def join_with_persons(self, group_population, persons_group_assignment, roles: I group_population.members_role = np.select([role_names_array == role.key for role in flattened_roles], flattened_roles) def build(self, tax_benefit_system): - return Simulation(tax_benefit_system, self.entities_instances) + return Simulation(tax_benefit_system, self.populations) def explicit_singular_entities(self, tax_benefit_system, input_dict): """ From 50150a542a56e34b06ac8c1e7c0c004dda179fb7 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 12:25:15 +0100 Subject: [PATCH 24/40] Normalize class name --- tests/core/test_reforms.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/test_reforms.py b/tests/core/test_reforms.py index 5d10086198..c76739691d 100644 --- a/tests/core/test_reforms.py +++ b/tests/core/test_reforms.py @@ -35,13 +35,13 @@ class goes_to_school(Variable): tax_benefit_system.add_variable(goes_to_school) -class test_basic_income_neutralization(Reform): +class WithBasicIncomeNeutralized(Reform): def apply(self): self.neutralize_variable('basic_income') def test_formula_neutralization(make_simulation): - reform = test_basic_income_neutralization(tax_benefit_system) + reform = WithBasicIncomeNeutralized(tax_benefit_system) period = '2017-01' simulation = make_simulation(reform.base_tax_benefit_system, period, {}) @@ -76,7 +76,7 @@ def apply(self): def test_neutralization_optimization(make_simulation): - reform = test_basic_income_neutralization(tax_benefit_system) + reform = WithBasicIncomeNeutralized(tax_benefit_system) period = '2017-01' simulation = make_simulation(reform, period, {}) From e3d5197f93fa5528ab700a5befd2fea5adddd12a Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 15:22:42 +0100 Subject: [PATCH 25/40] Respect TaxBenefitSystem constructor contract --- openfisca_core/reforms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfisca_core/reforms.py b/openfisca_core/reforms.py index 870dac5e89..75855836c2 100644 --- a/openfisca_core/reforms.py +++ b/openfisca_core/reforms.py @@ -42,6 +42,7 @@ def __init__(self, baseline): """ :param baseline: Baseline TaxBenefitSystem. """ + super().__init__(baseline.entities) self.baseline = baseline self.parameters = baseline.parameters self._parameters_at_instant_cache = baseline._parameters_at_instant_cache From 07d16c47f64571427ce3d3ef2fa595f2fd939bc3 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 15:37:45 +0100 Subject: [PATCH 26/40] Fix clone --- openfisca_core/taxbenefitsystems.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index 97d1bdd97e..44757fc6d8 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -444,6 +444,8 @@ def clone(self): for key, value in self.__dict__.items(): if key not in ('parameters', '_parameters_at_instant_cache', 'variables', 'open_api_config'): new_dict[key] = value + for entity in new_dict['entities']: + entity.set_tax_benefit_system(new) new_dict['parameters'] = self.parameters.clone() new_dict['_parameters_at_instant_cache'] = {} From dc61322385d4a1368d9428ebf6af3cf49be9794c Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 15:44:09 +0100 Subject: [PATCH 27/40] Fix dump and restore --- openfisca_core/tools/simulation_dumper.py | 52 +++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/openfisca_core/tools/simulation_dumper.py b/openfisca_core/tools/simulation_dumper.py index 24a4a26478..7a149106f4 100644 --- a/openfisca_core/tools/simulation_dumper.py +++ b/openfisca_core/tools/simulation_dumper.py @@ -42,16 +42,16 @@ def restore_simulation(directory, tax_benefit_system, **kwargs): simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) entities_dump_dir = os.path.join(directory, "__entities__") - for entity in simulation.entities.values(): - if entity.is_person: + for population in simulation.entities.values(): + if population.entity.is_person: continue - person_count = _restore_entity(entity, entities_dump_dir) + person_count = _restore_entity(population, entities_dump_dir) - for entity in simulation.entities.values(): - if not entity.is_person: + for population in simulation.entities.values(): + if not population.entity.is_person: continue - _restore_entity(entity, entities_dump_dir) - entity.count = person_count + _restore_entity(population, entities_dump_dir) + population.count = person_count variables_to_restore = (variable for variable in os.listdir(directory) if variable != "__entities__") for variable in variables_to_restore: @@ -67,40 +67,40 @@ def _dump_holder(holder, directory): disk_storage.put(value, period) -def _dump_entity(entity, directory): - path = os.path.join(directory, entity.key) +def _dump_entity(population, directory): + path = os.path.join(directory, population.entity.key) os.mkdir(path) - np.save(os.path.join(path, "id.npy"), entity.ids) + np.save(os.path.join(path, "id.npy"), population.ids) - if entity.is_person: + if population.entity.is_person: return - np.save(os.path.join(path, "members_position.npy"), entity.members_position) - np.save(os.path.join(path, "members_entity_id.npy"), entity.members_entity_id) + np.save(os.path.join(path, "members_position.npy"), population.members_position) + np.save(os.path.join(path, "members_entity_id.npy"), population.members_entity_id) encoded_roles = np.select( - [entity.members_role == role for role in entity.flattened_roles], - [role.key for role in entity.flattened_roles], + [population.members_role == role for role in population.entity.flattened_roles], + [role.key for role in population.entity.flattened_roles], ) np.save(os.path.join(path, "members_role.npy"), encoded_roles) -def _restore_entity(entity, directory): - path = os.path.join(directory, entity.key) +def _restore_entity(population, directory): + path = os.path.join(directory, population.entity.key) - entity.ids = np.load(os.path.join(path, "id.npy")) + population.ids = np.load(os.path.join(path, "id.npy")) - if entity.is_person: + if population.entity.is_person: return - entity.members_position = np.load(os.path.join(path, "members_position.npy")) - entity.members_entity_id = np.load(os.path.join(path, "members_entity_id.npy")) + population.members_position = np.load(os.path.join(path, "members_position.npy")) + population.members_entity_id = np.load(os.path.join(path, "members_entity_id.npy")) encoded_roles = np.load(os.path.join(path, "members_role.npy")) - entity.members_role = np.select( - [encoded_roles == role.key for role in entity.flattened_roles], - [role for role in entity.flattened_roles], + population.members_role = np.select( + [encoded_roles == role.key for role in population.entity.flattened_roles], + [role for role in population.entity.flattened_roles], ) - person_count = len(entity.members_entity_id) - entity.count = max(entity.members_entity_id) + 1 + person_count = len(population.members_entity_id) + population.count = max(population.members_entity_id) + 1 return person_count From b079dfdc36d3e286db3d20e810a7801c10cf016f Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Sun, 24 Mar 2019 20:10:44 +0100 Subject: [PATCH 28/40] Don't default dtype to float in filled_array --- openfisca_core/entities.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 21d61212aa..ba08e3780c 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -57,7 +57,7 @@ def empty_array(self): return np.zeros(self.count) def filled_array(self, value, dtype = None): - return np.full(self.count, value, dtype or float) + return np.full(self.count, value, dtype) def __getattr__(self, attribute): projector = get_projector_from_shortcut(self, attribute) @@ -381,7 +381,8 @@ def reduce(self, array, reducer, neutral_element, role = None): biggest_entity_size = np.max(position_in_entity) + 1 for p in range(biggest_entity_size): - result = reducer(result, self.value_nth_person(p, filtered_array, default = neutral_element)) + values = self.value_nth_person(p, filtered_array, default = neutral_element) + result = reducer(result, values) return result From 2b259585a2a180509a640629d76cc53e582e5df6 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 16:28:18 +0200 Subject: [PATCH 29/40] Remove obsolete to_json method --- openfisca_core/entities.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index ba08e3780c..16223bb8d9 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -217,16 +217,6 @@ def __init__(self, key, plural, label, doc): def set_tax_benefit_system(self, tax_benefit_system): self.tax_benefit_system = tax_benefit_system - def to_json(self): - return { - 'isPersonsEntity': self.is_person, - 'key': self.key, - 'label': self.label, - 'plural': self.plural, - 'doc': self.doc, - 'roles': self.roles_description, - } - def check_role_validity(self, role): if role is not None and not type(role) == Role: raise ValueError("{} is not a valid role".format(role)) From e0bf70c9984e6d423ea72d964f63767c347b12ab Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 16:57:38 +0200 Subject: [PATCH 30/40] Make Entity's tax_benefit_system private --- openfisca_core/entities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 16223bb8d9..5489fb4485 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -212,17 +212,17 @@ def __init__(self, key, plural, label, doc): self.plural = plural self.doc = textwrap.dedent(doc) self.is_person = True - self.tax_benefit_system = None + self._tax_benefit_system = None def set_tax_benefit_system(self, tax_benefit_system): - self.tax_benefit_system = tax_benefit_system + self._tax_benefit_system = tax_benefit_system def check_role_validity(self, role): if role is not None and not type(role) == Role: raise ValueError("{} is not a valid role".format(role)) def get_variable(self, variable_name, check_existence = False): - return self.tax_benefit_system.get_variable(variable_name, check_existence) + return self._tax_benefit_system.get_variable(variable_name, check_existence) def check_variable_defined_for_entity(self, variable_name): variable_entity = self.get_variable(variable_name, check_existence = True).entity From b831c051b7f3da66c0122f99dd20b91c656e1f55 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 17:25:41 +0200 Subject: [PATCH 31/40] Rename Simulation's 'entities' to 'populations' --- openfisca_core/entities.py | 4 ++-- .../scripts/measure_performances.py | 4 ++-- .../scripts/simulation_generator.py | 2 +- openfisca_core/simulation_builder.py | 4 ++-- openfisca_core/simulations.py | 22 +++++++++---------- openfisca_core/tools/simulation_dumper.py | 6 ++--- openfisca_core/tools/test_runner.py | 2 +- tests/core/test_simulations.py | 4 ++-- .../tools/test_runner/test_yaml_runner.py | 2 +- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 5489fb4485..2c0fafd9bf 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -609,8 +609,8 @@ def transform(self, result): def get_projector_from_shortcut(population, shortcut, parent = None): if population.entity.is_person: - if shortcut in population.simulation.entities: - entity_2 = population.simulation.entities[shortcut] + if shortcut in population.simulation.populations: + entity_2 = population.simulation.populations[shortcut] return EntityToPersonProjector(entity_2, parent) else: if shortcut == 'first_person': diff --git a/openfisca_core/scripts/measure_performances.py b/openfisca_core/scripts/measure_performances.py index 69338ff8d4..1d84ddd585 100644 --- a/openfisca_core/scripts/measure_performances.py +++ b/openfisca_core/scripts/measure_performances.py @@ -187,11 +187,11 @@ def formula(self, simulation, period): @timeit def check_revenu_disponible(year, city_code, expected_revenu_disponible): simulation = simulations.Simulation(period = periods.period(year), tax_benefit_system = tax_benefit_system) - famille = simulation.entities["famille"] + famille = simulation.populations["famille"] famille.count = 3 famille.roles_count = 2 famille.step_size = 1 - individu = simulation.entities["individu"] + individu = simulation.populations["individu"] individu.count = 6 individu.step_size = 2 simulation.get_or_new_holder("city_code").array = np.array([city_code, city_code, city_code]) diff --git a/openfisca_core/scripts/simulation_generator.py b/openfisca_core/scripts/simulation_generator.py index f2c9eca8fa..489f42356f 100644 --- a/openfisca_core/scripts/simulation_generator.py +++ b/openfisca_core/scripts/simulation_generator.py @@ -36,7 +36,7 @@ def make_simulation(tax_benefit_system, nb_persons, nb_groups, **kwargs): members_legacy_role[id_person] = legacy_role members_entity_id[id_person] = id_group - for entity in simulation.entities.values(): + for entity in simulation.populations.values(): if not entity.is_person: entity.members_entity_id = members_entity_id entity.count = nb_groups diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index c8e3e8b7bf..be22cf5436 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -117,7 +117,7 @@ def build_from_entities(self, tax_benefit_system, input_dict): for entity_class in tax_benefit_system.group_entities: try: - population = simulation.entities[entity_class.key] + population = simulation.populations[entity_class.key] self.finalize_variables_init(population) except PeriodMismatchError as e: self.raise_period_mismatch(population.entity, instances_json, e) @@ -158,7 +158,7 @@ def build_default_simulation(self, tax_benefit_system, count = 1): """ simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) - for population in simulation.entities.values(): + for population in simulation.populations.values(): population.count = count population.ids = np.array(range(count)) if not population.entity.is_person: diff --git a/openfisca_core/simulations.py b/openfisca_core/simulations.py index a202dbf91a..4d4067784f 100644 --- a/openfisca_core/simulations.py +++ b/openfisca_core/simulations.py @@ -49,8 +49,8 @@ def __init__( self.tax_benefit_system = tax_benefit_system assert tax_benefit_system is not None - self.entities = populations - self.persons = self.entities[tax_benefit_system.person_entity.key] + self.populations = populations + self.persons = self.populations[tax_benefit_system.person_entity.key] self.link_to_entities_instances() self.create_shortcuts() @@ -82,11 +82,11 @@ def trace(self, trace): self.tracer = None def link_to_entities_instances(self): - for _key, entity_instance in self.entities.items(): + for _key, entity_instance in self.populations.items(): entity_instance.simulation = self def create_shortcuts(self): - for _key, population in self.entities.items(): + for _key, population in self.populations.items(): # create shortcut simulation.person and simulation.household (for instance) setattr(self, population.entity.key, population) @@ -378,7 +378,7 @@ def get_memory_usage(self, variables = None): total_nb_bytes = 0, by_variable = {} ) - for entity in self.entities.values(): + for entity in self.populations.values(): entity_memory_usage = entity.get_memory_usage(variables = variables) result['total_nb_bytes'] += entity_memory_usage['total_nb_bytes'] result['by_variable'].update(entity_memory_usage['by_variable']) @@ -458,10 +458,10 @@ def set_input(self, variable_name, period, value): def get_variable_population(self, variable_name): variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True) - return self.entities[variable.entity.key] + return self.populations[variable.entity.key] def get_population(self, plural = None): - return next((population for population in self.entities.values() if population.entity.plural == plural), None) + return next((population for population in self.populations.values() if population.entity.plural == plural), None) def get_entity(self, plural = None): population = self.get_population(plural) @@ -472,7 +472,7 @@ def get_index(self, plural, id): return population.ids.index(id) def describe_entities(self): - return {population.entity.plural: population.ids for population in self.entities.values()} + return {population.entity.plural: population.ids for population in self.populations.values()} def clone(self, debug = False, trace = False): """ @@ -487,11 +487,11 @@ def clone(self, debug = False, trace = False): new.persons = self.persons.clone(new) setattr(new, new.persons.entity.key, new.persons) - new.entities = {new.persons.entity.key: new.persons} + new.populations = {new.persons.entity.key: new.persons} for entity in self.tax_benefit_system.group_entities: - population = self.entities[entity.key].clone(new) - new.entities[entity.key] = population + population = self.populations[entity.key].clone(new) + new.populations[entity.key] = population setattr(new, entity.key, population) # create shortcut simulation.household (for instance) new.debug = debug diff --git a/openfisca_core/tools/simulation_dumper.py b/openfisca_core/tools/simulation_dumper.py index 7a149106f4..5e40a0132b 100644 --- a/openfisca_core/tools/simulation_dumper.py +++ b/openfisca_core/tools/simulation_dumper.py @@ -26,7 +26,7 @@ def dump_simulation(simulation, directory): entities_dump_dir = os.path.join(directory, "__entities__") os.mkdir(entities_dump_dir) - for entity in simulation.entities.values(): + for entity in simulation.populations.values(): # Dump entity structure _dump_entity(entity, entities_dump_dir) @@ -42,12 +42,12 @@ def restore_simulation(directory, tax_benefit_system, **kwargs): simulation = Simulation(tax_benefit_system, tax_benefit_system.instantiate_entities()) entities_dump_dir = os.path.join(directory, "__entities__") - for population in simulation.entities.values(): + for population in simulation.populations.values(): if population.entity.is_person: continue person_count = _restore_entity(population, entities_dump_dir) - for population in simulation.entities.values(): + for population in simulation.populations.values(): if not population.entity.is_person: continue _restore_entity(population, entities_dump_dir) diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 8d8242bd65..87a6c7f191 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -254,7 +254,7 @@ def _run_test(simulation, test): for key, expected_value in output.items(): if tax_benefit_system.variables.get(key): # If key is a variable _check_variable(simulation, key, expected_value, test.get('period'), test) - elif simulation.entities.get(key): # If key is an entity singular + elif simulation.populations.get(key): # If key is an entity singular for variable_name, value in expected_value.items(): _check_variable(simulation, variable_name, value, test.get('period'), test) else: diff --git a/tests/core/test_simulations.py b/tests/core/test_simulations.py index 45f20cb8ae..05a8a14c53 100644 --- a/tests/core/test_simulations.py +++ b/tests/core/test_simulations.py @@ -48,8 +48,8 @@ def test_clone(): simulation_clone = simulation.clone() assert simulation != simulation_clone - for entity_id, entity in simulation.entities.items(): - assert entity != simulation_clone.entities[entity_id] + for entity_id, entity in simulation.populations.items(): + assert entity != simulation_clone.populations[entity_id] assert simulation.persons != simulation_clone.persons diff --git a/tests/core/tools/test_runner/test_yaml_runner.py b/tests/core/tools/test_runner/test_yaml_runner.py index f4aa22d6dd..0cb7db3321 100644 --- a/tests/core/tools/test_runner/test_yaml_runner.py +++ b/tests/core/tools/test_runner/test_yaml_runner.py @@ -30,7 +30,7 @@ def __init__(self, baseline): class Simulation: def __init__(self): self.tax_benefit_system = TaxBenefitSystem() - self.entities = {} + self.populations = {} def get_population(self, plural = None): return None From 56e6a0807d52cdc0f4294beee14c4d773761b9c7 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 17:42:13 +0200 Subject: [PATCH 32/40] Move get_index from Simulation to Population --- openfisca_core/entities.py | 3 +++ openfisca_core/simulations.py | 4 ---- openfisca_core/tools/test_runner.py | 2 +- openfisca_web_api/handlers.py | 3 ++- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 2c0fafd9bf..eb6d219315 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -65,6 +65,9 @@ def __getattr__(self, attribute): raise AttributeError("Entity {} has no attribute {}".format(self.entity.key, attribute)) return projector + def get_index(self, id): + return self.ids.index(id) + # Calculations def check_array_compatible_with_entity(self, array): diff --git a/openfisca_core/simulations.py b/openfisca_core/simulations.py index 4d4067784f..09869d57fb 100644 --- a/openfisca_core/simulations.py +++ b/openfisca_core/simulations.py @@ -467,10 +467,6 @@ def get_entity(self, plural = None): population = self.get_population(plural) return population and population.entity - def get_index(self, plural, id): - population = self.get_population(plural) - return population.ids.index(id) - def describe_entities(self): return {population.entity.plural: population.ids for population in self.populations.values()} diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 87a6c7f191..9036dd8912 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -262,7 +262,7 @@ def _run_test(simulation, test): if population is not None: # If key is an entity plural for instance_id, instance_values in expected_value.items(): for variable_name, value in instance_values.items(): - entity_index = simulation.get_index(key, instance_id) + entity_index = population.get_index(instance_id) _check_variable(simulation, variable_name, value, test.get('period'), test, entity_index) else: raise VariableNotFound(key, tax_benefit_system) diff --git a/openfisca_web_api/handlers.py b/openfisca_web_api/handlers.py index b65e3b37b7..3e70781a96 100644 --- a/openfisca_web_api/handlers.py +++ b/openfisca_web_api/handlers.py @@ -20,7 +20,8 @@ def calculate(tax_benefit_system, input_data): entity_plural, entity_id, variable_name, period = path.split('/') variable = tax_benefit_system.get_variable(variable_name) result = simulation.calculate(variable_name, period) - entity_index = simulation.get_index(entity_plural, entity_id) + population = simulation.get_population(entity_plural) + entity_index = population.get_index(entity_id) if variable.value_type == Enum: entity_result = result.decode()[entity_index].name From 5799d1268dfe0f80278d9785f0577b8441288d34 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 17:48:46 +0200 Subject: [PATCH 33/40] Remove snark --- openfisca_core/variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_core/variables.py b/openfisca_core/variables.py index ed9695b00f..9f5a7e6d6f 100644 --- a/openfisca_core/variables.py +++ b/openfisca_core/variables.py @@ -210,7 +210,7 @@ def set(self, attributes, attribute_name, required = False, allowed_values = Non return value def set_entity(self, entity): - # TODO - isinstance() won't work due to (ab)use of load_module to load tax_benefit_system + # TODO - isinstance() won't work due to use of load_module to load tax_benefit_system # Just trust the input in the meantime # if not isinstance(entity, entities.Entity): # raise ValueError("Invalid value '{}' for attribute 'entity' in variable '{}'. Must be an instance of Entity." From 51321d177e83d326b4de37bdee5c08ea059fb847 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 17:56:23 +0200 Subject: [PATCH 34/40] Improve a name --- tests/core/test_formulas.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/test_formulas.py b/tests/core/test_formulas.py index 5da10168a6..cc5f0e1888 100644 --- a/tests/core/test_formulas.py +++ b/tests/core/test_formulas.py @@ -50,8 +50,8 @@ def formula(person, period): # TaxBenefitSystem instance declared after formulas -ourtbs = CountryTaxBenefitSystem() -ourtbs.add_variables(choice, uses_multiplication, uses_switch) +our_tbs = CountryTaxBenefitSystem() +our_tbs.add_variables(choice, uses_multiplication, uses_switch) @fixture @@ -63,7 +63,7 @@ def month(): def simulation(month): builder = SimulationBuilder() builder.default_period = month - simulation = builder.build_from_variables(ourtbs, {'choice': np.random.randint(2, size = 1000) + 1}) + simulation = builder.build_from_variables(our_tbs, {'choice': np.random.randint(2, size = 1000) + 1}) simulation.debug = True return simulation From 2d9ab165f02c938777e7bf063fea736a4dd50da0 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 17:59:59 +0200 Subject: [PATCH 35/40] Don't commit publicly to either Entity or Population terms --- openfisca_core/entities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index eb6d219315..4c7eda88f7 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -62,7 +62,7 @@ def filled_array(self, value, dtype = None): def __getattr__(self, attribute): projector = get_projector_from_shortcut(self, attribute) if not projector: - raise AttributeError("Entity {} has no attribute {}".format(self.entity.key, attribute)) + raise AttributeError("You tried to use the '{}' of '{}' but that is not a known attribute.".format(attribute, self.entity.key)) return projector def get_index(self, id): From d7987703f5607004a5630d4c7f84b4a1f793a008 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Tue, 9 Apr 2019 18:23:31 +0200 Subject: [PATCH 36/40] Split Population classes into their own module --- openfisca_core/entities.py | 546 ------------------------- openfisca_core/model_api.py | 2 +- openfisca_core/populations.py | 550 ++++++++++++++++++++++++++ openfisca_core/simulation_builder.py | 3 +- openfisca_core/taxbenefitsystems.py | 3 +- tests/core/test_countries.py | 2 +- tests/core/test_simulation_builder.py | 3 +- 7 files changed, 558 insertions(+), 551 deletions(-) create mode 100644 openfisca_core/populations.py diff --git a/openfisca_core/entities.py b/openfisca_core/entities.py index 4c7eda88f7..f92009522e 100644 --- a/openfisca_core/entities.py +++ b/openfisca_core/entities.py @@ -2,24 +2,6 @@ import textwrap from os import linesep -from typing import Iterable - -import numpy as np - -from openfisca_core.indexed_enums import EnumArray -from openfisca_core.holders import Holder - -ADD = 'add' -DIVIDE = 'divide' - - -def projectable(function): - """ - Decorator to indicate that when called on a projector, the outcome of the function must be projected. - For instance person.household.sum(...) must be projected on person, while it would not make sense for person.household.get_holder. - """ - function.projectable = True - return function class Role(object): @@ -37,174 +19,6 @@ def __repr__(self): return "Role({})".format(self.key) -class Population(object): - def __init__(self, entity): - self.simulation = None - self.entity = entity - self._holders = {} - self.count = 0 - self.ids = [] - - def clone(self, simulation): - result = Population(self.entity) - result.simulation = simulation - result._holders = {variable: holder.clone(result) for (variable, holder) in self._holders.items()} - result.count = self.count - result.ids = self.ids - return result - - def empty_array(self): - return np.zeros(self.count) - - def filled_array(self, value, dtype = None): - return np.full(self.count, value, dtype) - - def __getattr__(self, attribute): - projector = get_projector_from_shortcut(self, attribute) - if not projector: - raise AttributeError("You tried to use the '{}' of '{}' but that is not a known attribute.".format(attribute, self.entity.key)) - return projector - - def get_index(self, id): - return self.ids.index(id) - - # Calculations - - def check_array_compatible_with_entity(self, array): - if not self.count == array.size: - raise ValueError("Input {} is not a valid value for the entity {} (size = {} != {} = count)".format( - array, self.key, array.size, self.count)) - - def __call__(self, variable_name, period, options = None, **parameters): - """ - Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. - - Example: - - >>> person('salary', '2017-04') - >>> array([300.]) - - :returns: A numpy array containing the result of the calculation - """ - self.entity.check_variable_defined_for_entity(variable_name) - - if options is None: - options = [] - - if ADD in options and DIVIDE in options: - raise ValueError('Options ADD and DIVIDE are incompatible (trying to compute variable {})'.format(variable_name).encode('utf-8')) - elif ADD in options: - return self.simulation.calculate_add(variable_name, period, **parameters) - elif DIVIDE in options: - return self.simulation.calculate_divide(variable_name, period, **parameters) - else: - return self.simulation.calculate(variable_name, period, **parameters) - - # Helpers - - def get_holder(self, variable_name): - self.entity.check_variable_defined_for_entity(variable_name) - holder = self._holders.get(variable_name) - if holder: - return holder - variable = self.entity.get_variable(variable_name) - self._holders[variable_name] = holder = Holder(variable, self) - return holder - - def get_memory_usage(self, variables = None): - holders_memory_usage = { - variable_name: holder.get_memory_usage() - for variable_name, holder in self._holders.items() - if variables is None or variable_name in variables - } - - total_memory_usage = sum( - holder_memory_usage['total_nb_bytes'] for holder_memory_usage in holders_memory_usage.values() - ) - - return dict( - total_nb_bytes = total_memory_usage, - by_variable = holders_memory_usage - ) - - @projectable - def has_role(self, role): - """ - Check if a person has a given role within its :any:`GroupEntity` - - Example: - - >>> person.has_role(Household.CHILD) - >>> array([False]) - """ - self.entity.check_role_validity(role) - group_population = self.simulation.get_population(role.entity.plural) - if role.subroles: - return np.logical_or.reduce([group_population.members_role == subrole for subrole in role.subroles]) - else: - return group_population.members_role == role - - @projectable - def value_from_partner(self, array, entity, role): - self.check_array_compatible_with_entity(array) - self.entity.check_role_validity(role) - - if not role.subroles or not len(role.subroles) == 2: - raise Exception('Projection to partner is only implemented for roles having exactly two subroles.') - - [subrole_1, subrole_2] = role.subroles - value_subrole_1 = entity.value_from_person(array, subrole_1) - value_subrole_2 = entity.value_from_person(array, subrole_2) - - return np.select( - [self.has_role(subrole_1), self.has_role(subrole_2)], - [value_subrole_2, value_subrole_1], - ) - - @projectable - def get_rank(self, entity, criteria, condition = True): - """ - Get the rank of a person within an entity according to a criteria. - The person with rank 0 has the minimum value of criteria. - If condition is specified, then the persons who don't respect it are not taken into account and their rank is -1. - - Example: - - >>> age = person('age', period) # e.g [32, 34, 2, 8, 1] - >>> person.get_rank(household, age) - >>> [3, 4, 0, 2, 1] - - >>> is_child = person.has_role(Household.CHILD) # [False, False, True, True, True] - >>> person.get_rank(household, - age, condition = is_child) # Sort in reverse order so that the eldest child gets the rank 0. - >>> [-1, -1, 1, 0, 2] - """ - - # If entity is for instance 'person.household', we get the reference entity 'household' behind the projector - entity = entity if not isinstance(entity, Projector) else entity.reference_entity - - positions = entity.members_position - biggest_entity_size = np.max(positions) + 1 - filtered_criteria = np.where(condition, criteria, np.inf) - ids = entity.members_entity_id - - # Matrix: the value in line i and column j is the value of criteria for the jth person of the ith entity - matrix = np.asarray([ - entity.value_nth_person(k, filtered_criteria, default = np.inf) - for k in range(biggest_entity_size) - ]).transpose() - - # We double-argsort all lines of the matrix. - # Double-argsorting gets the rank of each value once sorted - # For instance, if x = [3,1,6,4,0], y = np.argsort(x) is [4, 1, 0, 3, 2] (because the value with index 4 is the smallest one, the value with index 1 the second smallest, etc.) and z = np.argsort(y) is [2, 1, 4, 3, 0], the rank of each value. - sorted_matrix = np.argsort(np.argsort(matrix)) - - # Build the result vector by taking for each person the value in the right line (corresponding to its household id) and the right column (corresponding to its position) - result = sorted_matrix[ids, positions] - - # Return -1 for the persons who don't respect the condition - return np.where(condition, result, -1) - - class Entity(object): """ Represents an entity (e.g. a person, a household, etc.) on which calculations can be run. @@ -240,283 +54,6 @@ def check_variable_defined_for_entity(self, variable_name): raise ValueError(message) -class GroupPopulation(Population): - def __init__(self, entity, members): - super().__init__(entity) - self.members = members - self._members_entity_id = None - self._members_role = None - self._members_position = None - self._ordered_members_map = None - - def clone(self, simulation): - result = GroupPopulation(self.entity, self.members) - result.simulation = simulation - result._holders = {variable: holder.clone(self) for (variable, holder) in self._holders.items()} - result.count = self.count - result.ids = self.ids - result._members_entity_id = self._members_entity_id - result._members_role = self._members_role - result._members_position = self._members_position - result._ordered_members_map = self._ordered_members_map - return result - - @property - def members_position(self): - if self._members_position is None and self.members_entity_id is not None: - # We could use self.count and self.members.count , but with the current initilization, we are not sure count will be set before members_position is called - nb_entities = np.max(self.members_entity_id) + 1 - nb_persons = len(self.members_entity_id) - self._members_position = np.empty_like(self.members_entity_id) - counter_by_entity = np.zeros(nb_entities) - for k in range(nb_persons): - entity_index = self.members_entity_id[k] - self._members_position[k] = counter_by_entity[entity_index] - counter_by_entity[entity_index] += 1 - - return self._members_position - - @property - def members_entity_id(self): - return self._members_entity_id - - @members_entity_id.setter - def members_entity_id(self, members_entity_id): - self._members_entity_id = members_entity_id - - @property - def members_role(self): - if self._members_role is None: - default_role = self.entity.flattened_roles[0] - self._members_role = np.repeat(default_role, len(self.members_entity_id)) - return self._members_role - - @members_role.setter - def members_role(self, members_role: Iterable[Role]): - if members_role is not None: - self._members_role = np.array(list(members_role)) - - @property - def ordered_members_map(self): - """ - Mask to group the persons by entity - This function only caches the map value, to see what the map is used for, see value_nth_person method. - """ - if self._ordered_members_map is None: - return np.argsort(self.members_entity_id) - return self._ordered_members_map - - def get_role(self, role_name): - return next((role for role in self.entity.flattened_roles if role.key == role_name), None) - - @members_position.setter - def members_position(self, members_position): - self._members_position = members_position - - # Aggregation persons -> entity - - @projectable - def sum(self, array, role = None): - """ - Return the sum of ``array`` for the members of the entity. - - ``array`` must have the dimension of the number of persons in the simulation - - If ``role`` is provided, only the entity member with the given role are taken into account. - - Example: - - >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] - >>> household.sum(salaries) - >>> array([3500]) - """ - self.entity.check_role_validity(role) - self.members.check_array_compatible_with_entity(array) - if role is not None: - role_filter = self.members.has_role(role) - return np.bincount( - self.members_entity_id[role_filter], - weights = array[role_filter], - minlength = self.count) - else: - return np.bincount(self.members_entity_id, weights = array) - - @projectable - def any(self, array, role = None): - """ - Return ``True`` if ``array`` is ``True`` for any members of the entity. - - ``array`` must have the dimension of the number of persons in the simulation - - If ``role`` is provided, only the entity member with the given role are taken into account. - - Example: - - >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] - >>> household.any(salaries >= 1800) - >>> array([True]) - """ - sum_in_entity = self.sum(array, role = role) - return (sum_in_entity > 0) - - @projectable - def reduce(self, array, reducer, neutral_element, role = None): - self.members.check_array_compatible_with_entity(array) - self.entity.check_role_validity(role) - position_in_entity = self.members_position - role_filter = self.members.has_role(role) if role is not None else True - filtered_array = np.where(role_filter, array, neutral_element) - - result = self.filled_array(neutral_element) # Neutral value that will be returned if no one with the given role exists. - - # We loop over the positions in the entity - # Looping over the entities is tempting, but potentielly slow if there are a lot of entities - biggest_entity_size = np.max(position_in_entity) + 1 - - for p in range(biggest_entity_size): - values = self.value_nth_person(p, filtered_array, default = neutral_element) - result = reducer(result, values) - - return result - - @projectable - def all(self, array, role = None): - """ - Return ``True`` if ``array`` is ``True`` for all members of the entity. - - ``array`` must have the dimension of the number of persons in the simulation - - If ``role`` is provided, only the entity member with the given role are taken into account. - - Example: - - >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] - >>> household.all(salaries >= 1800) - >>> array([False]) - """ - return self.reduce(array, reducer = np.logical_and, neutral_element = True, role = role) - - @projectable - def max(self, array, role = None): - """ - Return the maximum value of ``array`` for the entity members. - - ``array`` must have the dimension of the number of persons in the simulation - - If ``role`` is provided, only the entity member with the given role are taken into account. - - Example: - - >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] - >>> household.max(salaries) - >>> array([2000]) - """ - return self.reduce(array, reducer = np.maximum, neutral_element = - np.infty, role = role) - - @projectable - def min(self, array, role = None): - """ - Return the minimum value of ``array`` for the entity members. - - ``array`` must have the dimension of the number of persons in the simulation - - If ``role`` is provided, only the entity member with the given role are taken into account. - - Example: - - >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] - >>> household.min(salaries) - >>> array([0]) - >>> household.min(salaries, role = Household.PARENT) # Assuming the 1st two persons are parents - >>> array([1500]) - """ - return self.reduce(array, reducer = np.minimum, neutral_element = np.infty, role = role) - - @projectable - def nb_persons(self, role = None): - """ - Returns the number of persons contained in the entity. - - If ``role`` is provided, only the entity member with the given role are taken into account. - """ - if role: - if role.subroles: - role_condition = np.logical_or.reduce([self.members_role == subrole for subrole in role.subroles]) - else: - role_condition = self.members_role == role - return self.sum(role_condition) - else: - return np.bincount(self.members_entity_id) - - # Projection person -> entity - - @projectable - def value_from_person(self, array, role, default = 0): - """ - Get the value of ``array`` for the person with the unique role ``role``. - - ``array`` must have the dimension of the number of persons in the simulation - - If such a person does not exist, return ``default`` instead - - The result is a vector which dimension is the number of entities - """ - self.entity.check_role_validity(role) - if role.max != 1: - raise Exception( - 'You can only use value_from_person with a role that is unique in {}. Role {} is not unique.' - .format(self.key, role.key) - ) - self.members.check_array_compatible_with_entity(array) - members_map = self.ordered_members_map - result = self.filled_array(default, dtype = array.dtype) - if isinstance(array, EnumArray): - result = EnumArray(result, array.possible_values) - role_filter = self.members.has_role(role) - entity_filter = self.any(role_filter) - - result[entity_filter] = array[members_map][role_filter[members_map]] - - return result - - @projectable - def value_nth_person(self, n, array, default = 0): - """ - Get the value of array for the person whose position in the entity is n. - - Note that this position is arbitrary, and that members are not sorted. - - If the nth person does not exist, return ``default`` instead. - - The result is a vector which dimension is the number of entities. - """ - self.members.check_array_compatible_with_entity(array) - positions = self.members_position - nb_persons_per_entity = self.nb_persons() - members_map = self.ordered_members_map - result = self.filled_array(default, dtype = array.dtype) - # For households that have at least n persons, set the result as the value of criteria for the person for which the position is n. - # The map is needed b/c the order of the nth persons of each household in the persons vector is not necessarily the same than the household order. - result[nb_persons_per_entity > n] = array[members_map][positions[members_map] == n] - - return result - - @projectable - def value_from_first_person(self, array): - return self.value_nth_person(0, array) - - # Projection entity -> person(s) - - def project(self, array, role = None): - self.check_array_compatible_with_entity(array) - self.entity.check_role_validity(role) - if role is None: - return array[self.members_entity_id] - else: - role_condition = self.members.has_role(role) - return np.where(role_condition, array[self.members_entity_id], 0) - - class GroupEntity(Entity): """ Represents an entity composed of several persons with different roles, on which calculations are run. @@ -540,89 +77,6 @@ def __init__(self, key, plural, label, doc, roles): self.is_person = False -class Projector(object): - reference_entity = None - parent = None - - def __getattr__(self, attribute): - projector = get_projector_from_shortcut(self.reference_entity, attribute, parent = self) - if projector: - return projector - - reference_attr = getattr(self.reference_entity, attribute) - if not hasattr(reference_attr, 'projectable'): - return reference_attr - - def projector_function(*args, **kwargs): - result = reference_attr(*args, **kwargs) - return self.transform_and_bubble_up(result) - - return projector_function - - def __call__(self, *args, **kwargs): - result = self.reference_entity(*args, **kwargs) - return self.transform_and_bubble_up(result) - - def transform_and_bubble_up(self, result): - transformed_result = self.transform(result) - if self.parent is None: - return transformed_result - else: - return self.parent.transform_and_bubble_up(transformed_result) - - def transform(self, result): - return NotImplementedError() - - -# For instance person.family -class EntityToPersonProjector(Projector): - - def __init__(self, entity, parent = None): - self.reference_entity = entity - self.parent = parent - - def transform(self, result): - return self.reference_entity.project(result) - - -# For instance famille.first_person -class FirstPersonToEntityProjector(Projector): - - def __init__(self, entity, parent = None): - self.target_entity = entity - self.reference_entity = entity.members - self.parent = parent - - def transform(self, result): - return self.target_entity.value_from_first_person(result) - - -# For instance famille.declarant_principal -class UniqueRoleToEntityProjector(Projector): - - def __init__(self, entity, role, parent = None): - self.target_entity = entity - self.reference_entity = entity.members - self.parent = parent - self.role = role - - def transform(self, result): - return self.target_entity.value_from_person(result, self.role) - - -def get_projector_from_shortcut(population, shortcut, parent = None): - if population.entity.is_person: - if shortcut in population.simulation.populations: - entity_2 = population.simulation.populations[shortcut] - return EntityToPersonProjector(entity_2, parent) - else: - if shortcut == 'first_person': - return FirstPersonToEntityProjector(population, parent) - role = next((role for role in population.entity.flattened_roles if (role.max == 1) and (role.key == shortcut)), None) - if role: - return UniqueRoleToEntityProjector(population, role, parent) - - def build_entity(key, plural, label, doc = "", roles = None, is_person = False, class_override = None): if is_person: return Entity(key, plural, label, doc) diff --git a/openfisca_core/model_api.py b/openfisca_core/model_api.py index 3b118e9ee6..567ea42b9e 100644 --- a/openfisca_core/model_api.py +++ b/openfisca_core/model_api.py @@ -16,7 +16,7 @@ set_input_divide_by_period, ) from openfisca_core.indexed_enums import Enum # noqa analysis:ignore -from openfisca_core.entities import (ADD, DIVIDE) # noqa analysis:ignore +from openfisca_core.populations import (ADD, DIVIDE) # noqa analysis:ignore from openfisca_core.simulations import ( # noqa analysis:ignore calculate_output_add, calculate_output_divide, diff --git a/openfisca_core/populations.py b/openfisca_core/populations.py new file mode 100644 index 0000000000..ee5ca49ec8 --- /dev/null +++ b/openfisca_core/populations.py @@ -0,0 +1,550 @@ +# -*- coding: utf-8 -*- + +from typing import Iterable + +import numpy as np + +from openfisca_core.entities import Role +from openfisca_core.indexed_enums import EnumArray +from openfisca_core.holders import Holder + + +ADD = 'add' +DIVIDE = 'divide' + + +def projectable(function): + """ + Decorator to indicate that when called on a projector, the outcome of the function must be projected. + For instance person.household.sum(...) must be projected on person, while it would not make sense for person.household.get_holder. + """ + function.projectable = True + return function + + +class Population(object): + def __init__(self, entity): + self.simulation = None + self.entity = entity + self._holders = {} + self.count = 0 + self.ids = [] + + def clone(self, simulation): + result = Population(self.entity) + result.simulation = simulation + result._holders = {variable: holder.clone(result) for (variable, holder) in self._holders.items()} + result.count = self.count + result.ids = self.ids + return result + + def empty_array(self): + return np.zeros(self.count) + + def filled_array(self, value, dtype = None): + return np.full(self.count, value, dtype) + + def __getattr__(self, attribute): + projector = get_projector_from_shortcut(self, attribute) + if not projector: + raise AttributeError("You tried to use the '{}' of '{}' but that is not a known attribute.".format(attribute, self.entity.key)) + return projector + + def get_index(self, id): + return self.ids.index(id) + + # Calculations + + def check_array_compatible_with_entity(self, array): + if not self.count == array.size: + raise ValueError("Input {} is not a valid value for the entity {} (size = {} != {} = count)".format( + array, self.key, array.size, self.count)) + + def __call__(self, variable_name, period, options = None, **parameters): + """ + Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. + + Example: + + >>> person('salary', '2017-04') + >>> array([300.]) + + :returns: A numpy array containing the result of the calculation + """ + self.entity.check_variable_defined_for_entity(variable_name) + + if options is None: + options = [] + + if ADD in options and DIVIDE in options: + raise ValueError('Options ADD and DIVIDE are incompatible (trying to compute variable {})'.format(variable_name).encode('utf-8')) + elif ADD in options: + return self.simulation.calculate_add(variable_name, period, **parameters) + elif DIVIDE in options: + return self.simulation.calculate_divide(variable_name, period, **parameters) + else: + return self.simulation.calculate(variable_name, period, **parameters) + + # Helpers + + def get_holder(self, variable_name): + self.entity.check_variable_defined_for_entity(variable_name) + holder = self._holders.get(variable_name) + if holder: + return holder + variable = self.entity.get_variable(variable_name) + self._holders[variable_name] = holder = Holder(variable, self) + return holder + + def get_memory_usage(self, variables = None): + holders_memory_usage = { + variable_name: holder.get_memory_usage() + for variable_name, holder in self._holders.items() + if variables is None or variable_name in variables + } + + total_memory_usage = sum( + holder_memory_usage['total_nb_bytes'] for holder_memory_usage in holders_memory_usage.values() + ) + + return dict( + total_nb_bytes = total_memory_usage, + by_variable = holders_memory_usage + ) + + @projectable + def has_role(self, role): + """ + Check if a person has a given role within its :any:`GroupEntity` + + Example: + + >>> person.has_role(Household.CHILD) + >>> array([False]) + """ + self.entity.check_role_validity(role) + group_population = self.simulation.get_population(role.entity.plural) + if role.subroles: + return np.logical_or.reduce([group_population.members_role == subrole for subrole in role.subroles]) + else: + return group_population.members_role == role + + @projectable + def value_from_partner(self, array, entity, role): + self.check_array_compatible_with_entity(array) + self.entity.check_role_validity(role) + + if not role.subroles or not len(role.subroles) == 2: + raise Exception('Projection to partner is only implemented for roles having exactly two subroles.') + + [subrole_1, subrole_2] = role.subroles + value_subrole_1 = entity.value_from_person(array, subrole_1) + value_subrole_2 = entity.value_from_person(array, subrole_2) + + return np.select( + [self.has_role(subrole_1), self.has_role(subrole_2)], + [value_subrole_2, value_subrole_1], + ) + + @projectable + def get_rank(self, entity, criteria, condition = True): + """ + Get the rank of a person within an entity according to a criteria. + The person with rank 0 has the minimum value of criteria. + If condition is specified, then the persons who don't respect it are not taken into account and their rank is -1. + + Example: + + >>> age = person('age', period) # e.g [32, 34, 2, 8, 1] + >>> person.get_rank(household, age) + >>> [3, 4, 0, 2, 1] + + >>> is_child = person.has_role(Household.CHILD) # [False, False, True, True, True] + >>> person.get_rank(household, - age, condition = is_child) # Sort in reverse order so that the eldest child gets the rank 0. + >>> [-1, -1, 1, 0, 2] + """ + + # If entity is for instance 'person.household', we get the reference entity 'household' behind the projector + entity = entity if not isinstance(entity, Projector) else entity.reference_entity + + positions = entity.members_position + biggest_entity_size = np.max(positions) + 1 + filtered_criteria = np.where(condition, criteria, np.inf) + ids = entity.members_entity_id + + # Matrix: the value in line i and column j is the value of criteria for the jth person of the ith entity + matrix = np.asarray([ + entity.value_nth_person(k, filtered_criteria, default = np.inf) + for k in range(biggest_entity_size) + ]).transpose() + + # We double-argsort all lines of the matrix. + # Double-argsorting gets the rank of each value once sorted + # For instance, if x = [3,1,6,4,0], y = np.argsort(x) is [4, 1, 0, 3, 2] (because the value with index 4 is the smallest one, the value with index 1 the second smallest, etc.) and z = np.argsort(y) is [2, 1, 4, 3, 0], the rank of each value. + sorted_matrix = np.argsort(np.argsort(matrix)) + + # Build the result vector by taking for each person the value in the right line (corresponding to its household id) and the right column (corresponding to its position) + result = sorted_matrix[ids, positions] + + # Return -1 for the persons who don't respect the condition + return np.where(condition, result, -1) + + +class GroupPopulation(Population): + def __init__(self, entity, members): + super().__init__(entity) + self.members = members + self._members_entity_id = None + self._members_role = None + self._members_position = None + self._ordered_members_map = None + + def clone(self, simulation): + result = GroupPopulation(self.entity, self.members) + result.simulation = simulation + result._holders = {variable: holder.clone(self) for (variable, holder) in self._holders.items()} + result.count = self.count + result.ids = self.ids + result._members_entity_id = self._members_entity_id + result._members_role = self._members_role + result._members_position = self._members_position + result._ordered_members_map = self._ordered_members_map + return result + + @property + def members_position(self): + if self._members_position is None and self.members_entity_id is not None: + # We could use self.count and self.members.count , but with the current initilization, we are not sure count will be set before members_position is called + nb_entities = np.max(self.members_entity_id) + 1 + nb_persons = len(self.members_entity_id) + self._members_position = np.empty_like(self.members_entity_id) + counter_by_entity = np.zeros(nb_entities) + for k in range(nb_persons): + entity_index = self.members_entity_id[k] + self._members_position[k] = counter_by_entity[entity_index] + counter_by_entity[entity_index] += 1 + + return self._members_position + + @property + def members_entity_id(self): + return self._members_entity_id + + @members_entity_id.setter + def members_entity_id(self, members_entity_id): + self._members_entity_id = members_entity_id + + @property + def members_role(self): + if self._members_role is None: + default_role = self.entity.flattened_roles[0] + self._members_role = np.repeat(default_role, len(self.members_entity_id)) + return self._members_role + + @members_role.setter + def members_role(self, members_role: Iterable[Role]): + if members_role is not None: + self._members_role = np.array(list(members_role)) + + @property + def ordered_members_map(self): + """ + Mask to group the persons by entity + This function only caches the map value, to see what the map is used for, see value_nth_person method. + """ + if self._ordered_members_map is None: + return np.argsort(self.members_entity_id) + return self._ordered_members_map + + def get_role(self, role_name): + return next((role for role in self.entity.flattened_roles if role.key == role_name), None) + + @members_position.setter + def members_position(self, members_position): + self._members_position = members_position + + # Aggregation persons -> entity + + @projectable + def sum(self, array, role = None): + """ + Return the sum of ``array`` for the members of the entity. + + ``array`` must have the dimension of the number of persons in the simulation + + If ``role`` is provided, only the entity member with the given role are taken into account. + + Example: + + >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] + >>> household.sum(salaries) + >>> array([3500]) + """ + self.entity.check_role_validity(role) + self.members.check_array_compatible_with_entity(array) + if role is not None: + role_filter = self.members.has_role(role) + return np.bincount( + self.members_entity_id[role_filter], + weights = array[role_filter], + minlength = self.count) + else: + return np.bincount(self.members_entity_id, weights = array) + + @projectable + def any(self, array, role = None): + """ + Return ``True`` if ``array`` is ``True`` for any members of the entity. + + ``array`` must have the dimension of the number of persons in the simulation + + If ``role`` is provided, only the entity member with the given role are taken into account. + + Example: + + >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] + >>> household.any(salaries >= 1800) + >>> array([True]) + """ + sum_in_entity = self.sum(array, role = role) + return (sum_in_entity > 0) + + @projectable + def reduce(self, array, reducer, neutral_element, role = None): + self.members.check_array_compatible_with_entity(array) + self.entity.check_role_validity(role) + position_in_entity = self.members_position + role_filter = self.members.has_role(role) if role is not None else True + filtered_array = np.where(role_filter, array, neutral_element) + + result = self.filled_array(neutral_element) # Neutral value that will be returned if no one with the given role exists. + + # We loop over the positions in the entity + # Looping over the entities is tempting, but potentielly slow if there are a lot of entities + biggest_entity_size = np.max(position_in_entity) + 1 + + for p in range(biggest_entity_size): + values = self.value_nth_person(p, filtered_array, default = neutral_element) + result = reducer(result, values) + + return result + + @projectable + def all(self, array, role = None): + """ + Return ``True`` if ``array`` is ``True`` for all members of the entity. + + ``array`` must have the dimension of the number of persons in the simulation + + If ``role`` is provided, only the entity member with the given role are taken into account. + + Example: + + >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] + >>> household.all(salaries >= 1800) + >>> array([False]) + """ + return self.reduce(array, reducer = np.logical_and, neutral_element = True, role = role) + + @projectable + def max(self, array, role = None): + """ + Return the maximum value of ``array`` for the entity members. + + ``array`` must have the dimension of the number of persons in the simulation + + If ``role`` is provided, only the entity member with the given role are taken into account. + + Example: + + >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] + >>> household.max(salaries) + >>> array([2000]) + """ + return self.reduce(array, reducer = np.maximum, neutral_element = - np.infty, role = role) + + @projectable + def min(self, array, role = None): + """ + Return the minimum value of ``array`` for the entity members. + + ``array`` must have the dimension of the number of persons in the simulation + + If ``role`` is provided, only the entity member with the given role are taken into account. + + Example: + + >>> salaries = household.members('salary', '2018-01') # e.g. [2000, 1500, 0, 0, 0] + >>> household.min(salaries) + >>> array([0]) + >>> household.min(salaries, role = Household.PARENT) # Assuming the 1st two persons are parents + >>> array([1500]) + """ + return self.reduce(array, reducer = np.minimum, neutral_element = np.infty, role = role) + + @projectable + def nb_persons(self, role = None): + """ + Returns the number of persons contained in the entity. + + If ``role`` is provided, only the entity member with the given role are taken into account. + """ + if role: + if role.subroles: + role_condition = np.logical_or.reduce([self.members_role == subrole for subrole in role.subroles]) + else: + role_condition = self.members_role == role + return self.sum(role_condition) + else: + return np.bincount(self.members_entity_id) + + # Projection person -> entity + + @projectable + def value_from_person(self, array, role, default = 0): + """ + Get the value of ``array`` for the person with the unique role ``role``. + + ``array`` must have the dimension of the number of persons in the simulation + + If such a person does not exist, return ``default`` instead + + The result is a vector which dimension is the number of entities + """ + self.entity.check_role_validity(role) + if role.max != 1: + raise Exception( + 'You can only use value_from_person with a role that is unique in {}. Role {} is not unique.' + .format(self.key, role.key) + ) + self.members.check_array_compatible_with_entity(array) + members_map = self.ordered_members_map + result = self.filled_array(default, dtype = array.dtype) + if isinstance(array, EnumArray): + result = EnumArray(result, array.possible_values) + role_filter = self.members.has_role(role) + entity_filter = self.any(role_filter) + + result[entity_filter] = array[members_map][role_filter[members_map]] + + return result + + @projectable + def value_nth_person(self, n, array, default = 0): + """ + Get the value of array for the person whose position in the entity is n. + + Note that this position is arbitrary, and that members are not sorted. + + If the nth person does not exist, return ``default`` instead. + + The result is a vector which dimension is the number of entities. + """ + self.members.check_array_compatible_with_entity(array) + positions = self.members_position + nb_persons_per_entity = self.nb_persons() + members_map = self.ordered_members_map + result = self.filled_array(default, dtype = array.dtype) + # For households that have at least n persons, set the result as the value of criteria for the person for which the position is n. + # The map is needed b/c the order of the nth persons of each household in the persons vector is not necessarily the same than the household order. + result[nb_persons_per_entity > n] = array[members_map][positions[members_map] == n] + + return result + + @projectable + def value_from_first_person(self, array): + return self.value_nth_person(0, array) + + # Projection entity -> person(s) + + def project(self, array, role = None): + self.check_array_compatible_with_entity(array) + self.entity.check_role_validity(role) + if role is None: + return array[self.members_entity_id] + else: + role_condition = self.members.has_role(role) + return np.where(role_condition, array[self.members_entity_id], 0) + + +class Projector(object): + reference_entity = None + parent = None + + def __getattr__(self, attribute): + projector = get_projector_from_shortcut(self.reference_entity, attribute, parent = self) + if projector: + return projector + + reference_attr = getattr(self.reference_entity, attribute) + if not hasattr(reference_attr, 'projectable'): + return reference_attr + + def projector_function(*args, **kwargs): + result = reference_attr(*args, **kwargs) + return self.transform_and_bubble_up(result) + + return projector_function + + def __call__(self, *args, **kwargs): + result = self.reference_entity(*args, **kwargs) + return self.transform_and_bubble_up(result) + + def transform_and_bubble_up(self, result): + transformed_result = self.transform(result) + if self.parent is None: + return transformed_result + else: + return self.parent.transform_and_bubble_up(transformed_result) + + def transform(self, result): + return NotImplementedError() + + +# For instance person.family +class EntityToPersonProjector(Projector): + + def __init__(self, entity, parent = None): + self.reference_entity = entity + self.parent = parent + + def transform(self, result): + return self.reference_entity.project(result) + + +# For instance famille.first_person +class FirstPersonToEntityProjector(Projector): + + def __init__(self, entity, parent = None): + self.target_entity = entity + self.reference_entity = entity.members + self.parent = parent + + def transform(self, result): + return self.target_entity.value_from_first_person(result) + + +# For instance famille.declarant_principal +class UniqueRoleToEntityProjector(Projector): + + def __init__(self, entity, role, parent = None): + self.target_entity = entity + self.reference_entity = entity.members + self.parent = parent + self.role = role + + def transform(self, result): + return self.target_entity.value_from_person(result, self.role) + + +def get_projector_from_shortcut(population, shortcut, parent = None): + if population.entity.is_person: + if shortcut in population.simulation.populations: + entity_2 = population.simulation.populations[shortcut] + return EntityToPersonProjector(entity_2, parent) + else: + if shortcut == 'first_person': + return FirstPersonToEntityProjector(population, parent) + role = next((role for role in population.entity.flattened_roles if (role.max == 1) and (role.key == shortcut)), None) + if role: + return UniqueRoleToEntityProjector(population, role, parent) diff --git a/openfisca_core/simulation_builder.py b/openfisca_core/simulation_builder.py index be22cf5436..a481b45e83 100644 --- a/openfisca_core/simulation_builder.py +++ b/openfisca_core/simulation_builder.py @@ -5,7 +5,8 @@ import numpy as np from copy import deepcopy -from openfisca_core.entities import Entity, Population +from openfisca_core.entities import Entity +from openfisca_core.populations import Population from openfisca_core.variables import Variable from openfisca_core.errors import VariableNotFound, SituationParsingError, PeriodMismatchError diff --git a/openfisca_core/taxbenefitsystems.py b/openfisca_core/taxbenefitsystems.py index 44757fc6d8..2225249620 100644 --- a/openfisca_core/taxbenefitsystems.py +++ b/openfisca_core/taxbenefitsystems.py @@ -14,7 +14,8 @@ import copy from openfisca_core import periods -from openfisca_core.entities import Entity, Population, GroupPopulation +from openfisca_core.entities import Entity +from openfisca_core.populations import Population, GroupPopulation from openfisca_core.parameters import ParameterNode from openfisca_core.variables import Variable, get_neutralized_variable from openfisca_core.errors import VariableNotFound diff --git a/tests/core/test_countries.py b/tests/core/test_countries.py index 307490a195..e7e8ec0ee1 100644 --- a/tests/core/test_countries.py +++ b/tests/core/test_countries.py @@ -7,7 +7,7 @@ from openfisca_core.simulation_builder import SimulationBuilder from openfisca_core.taxbenefitsystems import VariableNameConflict, VariableNotFound from openfisca_core import periods -from openfisca_core.entities import DIVIDE +from openfisca_core.populations import DIVIDE from openfisca_country_template import CountryTaxBenefitSystem from openfisca_core.tools import assert_near diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 4f489dd6a3..3ef1e06dcb 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -10,7 +10,8 @@ from openfisca_core.simulation_builder import SimulationBuilder, Simulation from openfisca_core.tools import assert_near from openfisca_core.tools.test_runner import yaml -from openfisca_core.entities import Entity, GroupEntity, Population +from openfisca_core.entities import Entity, GroupEntity +from openfisca_core.populations import Population from openfisca_core.variables import Variable from openfisca_country_template.entities import Household from openfisca_country_template.situation_examples import couple From 55f7f4d47424a5393fc6122176b16092a582c3a0 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Wed, 10 Apr 2019 09:56:41 -0400 Subject: [PATCH 37/40] Add back type check on variable entity --- openfisca_core/variables.py | 8 +++----- tests/core/test_simulation_builder.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/openfisca_core/variables.py b/openfisca_core/variables.py index 9f5a7e6d6f..fe98b31a42 100644 --- a/openfisca_core/variables.py +++ b/openfisca_core/variables.py @@ -10,6 +10,7 @@ from datetime import date from openfisca_core import periods +from openfisca_core.entities import Entity from openfisca_core.indexed_enums import Enum, EnumArray, ENUM_ARRAY_DTYPE from openfisca_core.periods import DAY, MONTH, YEAR, ETERNITY from openfisca_core.tools import eval_expression @@ -210,11 +211,8 @@ def set(self, attributes, attribute_name, required = False, allowed_values = Non return value def set_entity(self, entity): - # TODO - isinstance() won't work due to use of load_module to load tax_benefit_system - # Just trust the input in the meantime - # if not isinstance(entity, entities.Entity): - # raise ValueError("Invalid value '{}' for attribute 'entity' in variable '{}'. Must be an instance of Entity." - # .format(entity, self.name).encode('utf-8')) + if not isinstance(entity, Entity): + raise ValueError(f"Invalid value '{entity}' for attribute 'entity' in variable '{self.name}'. Must be an instance of Entity.") return entity def set_possible_values(self, possible_values): diff --git a/tests/core/test_simulation_builder.py b/tests/core/test_simulation_builder.py index 3ef1e06dcb..195dd97286 100644 --- a/tests/core/test_simulation_builder.py +++ b/tests/core/test_simulation_builder.py @@ -43,7 +43,7 @@ def int_variable(persons): class intvar(Variable): definition_period = ETERNITY value_type = int - entity = persons.__class__ + entity = persons def __init__(self): super().__init__() @@ -57,7 +57,7 @@ def date_variable(persons): class datevar(Variable): definition_period = ETERNITY value_type = date - entity = persons.__class__ + entity = persons def __init__(self): super().__init__() From 5a0e1f0c663704dce9573b8a06196cc3f1cf7683 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Wed, 10 Apr 2019 16:09:26 +0200 Subject: [PATCH 38/40] Reinstate check for missing period argument --- openfisca_core/populations.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/openfisca_core/populations.py b/openfisca_core/populations.py index ee5ca49ec8..76ae1d2a91 100644 --- a/openfisca_core/populations.py +++ b/openfisca_core/populations.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import traceback + from typing import Iterable import numpy as np @@ -60,7 +62,19 @@ def check_array_compatible_with_entity(self, array): raise ValueError("Input {} is not a valid value for the entity {} (size = {} != {} = count)".format( array, self.key, array.size, self.count)) - def __call__(self, variable_name, period, options = None, **parameters): + def check_period_validity(self, variable_name, period): + if period is None: + stack = traceback.extract_stack() + filename, line_number, function_name, line_of_code = stack[-3] + raise ValueError(''' +You requested computation of variable "{}", but you did not specify on which period in "{}:{}": + {} +When you request the computation of a variable within a formula, you must always specify the period as the second parameter. The convention is to call this parameter "period". For example: + computed_salary = person('salary', period). +See more information at . +'''.format(variable_name, filename, line_number, line_of_code).encode('utf-8')) + + def __call__(self, variable_name, period = None, options = None, **parameters): """ Calculate the variable ``variable_name`` for the entity and the period ``period``, using the variable formula if it exists. @@ -72,6 +86,7 @@ def __call__(self, variable_name, period, options = None, **parameters): :returns: A numpy array containing the result of the calculation """ self.entity.check_variable_defined_for_entity(variable_name) + self.check_period_validity(variable_name, period) if options is None: options = [] From be3895c4cbb0d308b95a8337022efa9fc353cd78 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Wed, 10 Apr 2019 16:36:09 +0200 Subject: [PATCH 39/40] Remove Python 2 adaptation Co-Authored-By: Morendil --- openfisca_core/populations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_core/populations.py b/openfisca_core/populations.py index 76ae1d2a91..99ad93297f 100644 --- a/openfisca_core/populations.py +++ b/openfisca_core/populations.py @@ -72,7 +72,7 @@ def check_period_validity(self, variable_name, period): When you request the computation of a variable within a formula, you must always specify the period as the second parameter. The convention is to call this parameter "period". For example: computed_salary = person('salary', period). See more information at . -'''.format(variable_name, filename, line_number, line_of_code).encode('utf-8')) +'''.format(variable_name, filename, line_number, line_of_code)) def __call__(self, variable_name, period = None, options = None, **parameters): """ From f01bff4e004d166aac16f0da793a9e16136d6ae5 Mon Sep 17 00:00:00 2001 From: Laurent Bossavit Date: Fri, 12 Apr 2019 10:54:35 +0200 Subject: [PATCH 40/40] Bump version number --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa3c215252..9fdfc57592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +# 32.0.0 [#857](https://github.com/openfisca/openfisca-core/pull/857) + +### Breaking changes + +- Split the "Entity" class hierarchy (Entity, PersonEntity, GroupEntity) into two parallel hierarchies, representing respectively the abstract, model-level information (classes named Entity etc.) and the population-level information (classes named Population and GroupPopulation) + - As a result, the first parameter passed to a formula is now a Population instance + - Much more detail (and class diagrams) in the PR description +- Remove support from the syntax `some_entity.SOME_ROLE` to access roles (where `some_entity` is the entity passed to a formula). + +### Migration details + +- Use the standard SomeEntity.SOME_ROLE instead. (Where SomeEntity is the capitalized entity or instance, Household.PARENT.) +- Code that relied excessively on internal implementation details of Entity may break, and should be updated to access methods of Entity/Population instead. + # 31.0.1 [#840](https://github.com/openfisca/openfisca-core/pull/840) - Improve usability of Enum values: diff --git a/setup.py b/setup.py index 797d9a57b1..5a373ba474 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ setup( name = 'OpenFisca-Core', - version = '31.0.1', + version = '32.0.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [