From 56b760615dd37bcf90540e36eaf18387bfa9d424 Mon Sep 17 00:00:00 2001 From: Matthew Richards Date: Tue, 20 Oct 2020 10:21:19 +0000 Subject: [PATCH] #154: Move common.models to common.database - The only model in common.models was for the database backend (there's no model for the Python ICAT backend), hence the change - This commit also includes the changes required to correctly import the file from the new location - Updated the file tree in the README --- README.md | 7 +- common/database/backend.py | 2 +- common/database/filters.py | 8 +-- common/database/helpers.py | 4 +- .../db_models.py => database/models.py} | 0 common/icat/backend.py | 1 - common/models/__init__.py | 0 src/resources/entities/entity_map.py | 2 +- .../non_entities/sessions_endpoints.py | 2 +- test/test_entityHelper.py | 2 +- test/test_helpers.py | 2 +- util/icat_db_generator.py | 68 +++++++++---------- 12 files changed, 47 insertions(+), 51 deletions(-) rename common/{models/db_models.py => database/models.py} (100%) delete mode 100644 common/models/__init__.py diff --git a/README.md b/README.md index 3eeffab9..ba264d7f 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,9 @@ This is illustrated below. │ ├── database │ │ ├── backend.py │ │ ├── filters.py - │ │ └── helpers.py + │ │ ├── helpers.py + │ │ └── models.py │ ├── icat - │ ├── models - │ │ └── db_models.py │ ├── backends.py │ ├── constants.py │ ├── exceptions.py @@ -134,7 +133,7 @@ session endpoint. #### Mapped classes -The classes mapped from the database are stored in `/common/models/db_models.py`. Each model was +The classes mapped from the database are stored in `/common/database/models.py`. Each model was automatically generated using sqlacodegen. A class `EntityHelper` is defined so that each model may inherit two methods `to_dict()` and `update_from_dict(dictionary)`, both used for returning entities and updating them, in a form easily converted to JSON. diff --git a/common/database/backend.py b/common/database/backend.py index 7fdaa699..af01bc51 100644 --- a/common/database/backend.py +++ b/common/database/backend.py @@ -16,7 +16,7 @@ requires_session_id, ) from common.helpers import queries_records -from common.models.db_models import SESSION +from common.database.models import SESSION import uuid from common.exceptions import AuthenticationError import datetime diff --git a/common/database/filters.py b/common/database/filters.py index 70dceee0..8eba4e36 100644 --- a/common/database/filters.py +++ b/common/database/filters.py @@ -7,7 +7,7 @@ IncludeFilter, ) from common.exceptions import FilterError, MultipleIncludeError -from common.models import db_models +from common.database import models from sqlalchemy import asc, desc import logging @@ -57,15 +57,15 @@ def apply_filter(self, query): ) if self.included_included_field: - included_table = getattr(db_models, self.field) - included_included_table = getattr(db_models, self.included_field) + included_table = getattr(models, self.field) + included_included_table = getattr(models, self.included_field) query.base_query = query.base_query.join(included_table).join( included_included_table ) field = getattr(included_included_table, self.included_included_field) elif self.included_field: - included_table = getattr(db_models, self.field) + included_table = getattr(models, self.field) query.base_query = query.base_query.join(included_table) field = getattr(included_table, self.included_field) diff --git a/common/database/helpers.py b/common/database/helpers.py index e5a5aae6..8c64b7dd 100644 --- a/common/database/helpers.py +++ b/common/database/helpers.py @@ -13,9 +13,7 @@ BadRequestError, MultipleIncludeError, ) -from common.models import db_models -from common.models.db_models import ( - INVESTIGATIONUSER, +from common.database.models import ( INVESTIGATION, INSTRUMENT, FACILITYCYCLE, diff --git a/common/models/db_models.py b/common/database/models.py similarity index 100% rename from common/models/db_models.py rename to common/database/models.py diff --git a/common/icat/backend.py b/common/icat/backend.py index e315b8d3..ea5d4669 100644 --- a/common/icat/backend.py +++ b/common/icat/backend.py @@ -26,7 +26,6 @@ from common.config import config from common.exceptions import AuthenticationError -from common.models.db_models import SESSION log = logging.getLogger() diff --git a/common/models/__init__.py b/common/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/resources/entities/entity_map.py b/src/resources/entities/entity_map.py index e1939444..58894693 100644 --- a/src/resources/entities/entity_map.py +++ b/src/resources/entities/entity_map.py @@ -1,4 +1,4 @@ -from common.models.db_models import ( +from common.database.models import ( APPLICATION, DATACOLLECTIONDATAFILE, DATACOLLECTIONPARAMETER, diff --git a/src/resources/non_entities/sessions_endpoints.py b/src/resources/non_entities/sessions_endpoints.py index 492700e1..3d4202cf 100644 --- a/src/resources/non_entities/sessions_endpoints.py +++ b/src/resources/non_entities/sessions_endpoints.py @@ -10,7 +10,7 @@ get_row_by_id, ) from common.helpers import get_session_id_from_auth_header -from common.models.db_models import SESSION +from common.database.models import SESSION from common.backends import backend from common.exceptions import AuthenticationError diff --git a/test/test_entityHelper.py b/test/test_entityHelper.py index 780bc2d7..586ba40c 100644 --- a/test/test_entityHelper.py +++ b/test/test_entityHelper.py @@ -1,7 +1,7 @@ import datetime from unittest import TestCase -from common.models.db_models import DATAFILE, DATASET, DATAFILEFORMAT, INVESTIGATION +from common.database.models import DATAFILE, DATASET, DATAFILEFORMAT, INVESTIGATION class TestEntityHelper(TestCase): diff --git a/test/test_helpers.py b/test/test_helpers.py index ce4d517b..b91d10c6 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -25,7 +25,7 @@ get_session_id_from_auth_header, get_filters_from_query_string, ) -from common.models.db_models import SESSION +from common.database.models import SESSION from test.test_base import FlaskAppTest diff --git a/util/icat_db_generator.py b/util/icat_db_generator.py index 8c559633..23c6895f 100644 --- a/util/icat_db_generator.py +++ b/util/icat_db_generator.py @@ -6,7 +6,7 @@ from faker import Faker -from common.models import db_models +from common.database import models from common.session_manager import session_manager parser = argparse.ArgumentParser() @@ -123,7 +123,7 @@ class FacilityGenerator(Generator): amount = 1 def generate(self): - facility = db_models.FACILITY() + facility = models.FACILITY() facility.CREATE_ID = "user" facility.MOD_ID = "user" facility.MOD_TIME = get_date_time() @@ -143,7 +143,7 @@ def generate(self): @staticmethod def generate_data_collection(i): - data_collection = db_models.DATACOLLECTION() + data_collection = models.DATACOLLECTION() apply_common_attributes(data_collection, i) post_entity(data_collection) @@ -157,7 +157,7 @@ def generate(self): @staticmethod def generate_applications(i): - application = db_models.APPLICATION() + application = models.APPLICATION() apply_common_attributes(application, i) application.VERSION = randrange(1, 4) post_entity(application) @@ -172,7 +172,7 @@ def generate(self): @staticmethod def generate_dataset_type(i): - dataset_type = db_models.DATASETTYPE() + dataset_type = models.DATASETTYPE() apply_common_attributes(dataset_type, i) post_entity(dataset_type) @@ -186,7 +186,7 @@ def generate(self): @staticmethod def generate_facility_cycle(i): - facility_cycle = db_models.FACILITYCYCLE() + facility_cycle = models.FACILITYCYCLE() apply_common_attributes(facility_cycle, i) # overwrite the name with a more suitable one k = i % 4 + 1 # This will give a repeated 1, 2, 3, 4 for 4 cycles per year @@ -206,7 +206,7 @@ def generate(self): @staticmethod def generate_sample_type(i): - sample_type = db_models.SAMPLETYPE() + sample_type = models.SAMPLETYPE() apply_common_attributes(sample_type, i) sample_type.MOLECULARFORMULA = randrange(43, 13323) sample_type.SAFETYINFORMATION = faker.text() @@ -222,7 +222,7 @@ def generate(self): @staticmethod def generate_instruments(i): - instrument = db_models.INSTRUMENT() + instrument = models.INSTRUMENT() apply_common_attributes(instrument, i) instrument.FULLNAME = faker.text() instrument.URL = faker.url() @@ -239,7 +239,7 @@ def generate(self): @staticmethod def generate_users(i): - user = db_models.USER() + user = models.USER() apply_common_attributes(user, i) user.EMAIL = faker.ascii_email() user.NAME = faker.first_name() + f"{i}" @@ -257,7 +257,7 @@ def generate(self): @staticmethod def generate_datafile_format(i): - datafile_format = db_models.DATAFILEFORMAT() + datafile_format = models.DATAFILEFORMAT() apply_common_attributes(datafile_format, i) datafile_format.VERSION = randrange(1, 14) post_entity(datafile_format) @@ -272,7 +272,7 @@ def generate(self): @staticmethod def generate_investigation_type(i): - investigation_type = db_models.INVESTIGATIONTYPE() + investigation_type = models.INVESTIGATIONTYPE() apply_common_attributes(investigation_type, i) post_entity(investigation_type) @@ -286,7 +286,7 @@ def generate(self): @staticmethod def generate_groupings(i): - grouping = db_models.GROUPING() + grouping = models.GROUPING() apply_common_attributes(grouping, i) post_entity(grouping) @@ -300,7 +300,7 @@ def generate(self): @staticmethod def generate_investigations(i): - investigation = db_models.INVESTIGATION() + investigation = models.INVESTIGATION() apply_common_attributes(investigation, i) k = i % 4 + 1 year = 2000 + (i % 80) // 4 @@ -323,7 +323,7 @@ def generate(self): @staticmethod def generate_investigation_user(i): - investigation_user = db_models.INVESTIGATIONUSER() + investigation_user = models.INVESTIGATIONUSER() apply_common_attributes(investigation_user, i) investigation_user.ROLE = ["PI", "CI"][randrange(2)] investigation_user.INVESTIGATION_ID = i @@ -340,7 +340,7 @@ def generate(self): @staticmethod def generate_instrument_scientist(i): - instrument_scientist = db_models.INSTRUMENTSCIENTIST() + instrument_scientist = models.INSTRUMENTSCIENTIST() apply_common_attributes(instrument_scientist, i) instrument_scientist.INSTRUMENT_ID = i instrument_scientist.USER_ID = randrange(1, UserGenerator.amount) @@ -358,7 +358,7 @@ def generate(self): @staticmethod def generate_investigation_instrument(i): - investigation_instrument = db_models.INVESTIGATIONINSTRUMENT() + investigation_instrument = models.INVESTIGATIONINSTRUMENT() apply_common_attributes(investigation_instrument, i) investigation_instrument.INVESTIGATION_ID = i investigation_instrument.INSTRUMENT_ID = randrange(1, 15) @@ -374,7 +374,7 @@ def generate(self): @staticmethod def generate_sample(i): - sample = db_models.SAMPLE() + sample = models.SAMPLE() apply_common_attributes(sample, i) sample.INVESTIGATION_ID = i sample.SAMPLETYPE_ID = randrange(1, SampleTypeGenerator.amount) @@ -390,7 +390,7 @@ def generate(self): @staticmethod def generate_user_groups(i): - user_group = db_models.USERGROUP() + user_group = models.USERGROUP() apply_common_attributes(user_group, i) user_group.GROUP_ID = randrange(1, GroupingGenerator.amount) user_group.USER_ID = i @@ -406,7 +406,7 @@ def generate(self): @staticmethod def generate_studies(i): - study = db_models.STUDY() + study = models.STUDY() apply_common_attributes(study, i) study.STARTDATE = get_start_date(i) study.STATUS = randrange(2) @@ -423,7 +423,7 @@ def generate(self): @staticmethod def generate_investigation_group(i): - investigation_group = db_models.INVESTIGATIONGROUP() + investigation_group = models.INVESTIGATIONGROUP() apply_common_attributes(investigation_group, i) investigation_group.ROLE = faker.text() + str(i) investigation_group.GROUP_ID = randrange(1, GroupingGenerator.amount) @@ -440,7 +440,7 @@ def generate(self): @staticmethod def generate_keyword(i): - keyword = db_models.KEYWORD() + keyword = models.KEYWORD() apply_common_attributes(keyword, i) keyword.NAME = faker.word() + str(i) keyword.INVESTIGATION_ID = randrange(1, InvestigationGenerator.amount) @@ -456,7 +456,7 @@ def generate(self): @staticmethod def generate_publication(i): - publication = db_models.PUBLICATION() + publication = models.PUBLICATION() apply_common_attributes(publication, i) publication.FULLREFERENCE = faker.text() publication.REPOSITORY = faker.uri() @@ -475,7 +475,7 @@ def generate(self): @staticmethod def generate_parameter_type(i): - parameter_type = db_models.PARAMETERTYPE() + parameter_type = models.PARAMETERTYPE() apply_common_attributes(parameter_type, i) parameter_type.APPLICABLETODATACOLLECTION = randrange(2) parameter_type.APPLICABLETODATAFILE = randrange(2) @@ -486,7 +486,7 @@ def generate_parameter_type(i): parameter_type.MAXIMUMNUMERICVALUE = randrange(10) parameter_type.UNITS = f"unit {i}" parameter_type.UNITSFULLNAME = faker.word() - parameter_type.VALUETYPE = choice(list(db_models.PARAMETERTYPE.ValueTypeEnum)) + parameter_type.VALUETYPE = choice(list(models.PARAMETERTYPE.ValueTypeEnum)) parameter_type.VERIFIED = randrange(2) post_entity(parameter_type) @@ -500,7 +500,7 @@ def generate(self): def generate_investigation_parameter(i): - investigation_parameter = db_models.INVESTIGATIONPARAMETER() + investigation_parameter = models.INVESTIGATIONPARAMETER() apply_common_attributes(investigation_parameter, i) apply_common_parameter_attributes(investigation_parameter, i) investigation_parameter.INVESTIGATION_ID = i @@ -519,7 +519,7 @@ def generate(self): @staticmethod def generate_shift(i): - shift = db_models.SHIFT() + shift = models.SHIFT() apply_common_attributes(shift, i) shift.COMMENT = faker.text() shift.INVESTIGATION_ID = i @@ -535,7 +535,7 @@ def generate(self): @staticmethod def generate_study_investigation(i): - study_investigation = db_models.STUDYINVESTIGATION() + study_investigation = models.STUDYINVESTIGATION() apply_common_attributes(study_investigation, i) study_investigation.INVESTIGATION_ID = i study_investigation.STUDY_ID = randrange(1, StudyGenerator.amount) @@ -551,7 +551,7 @@ def generate(self): @staticmethod def generate_dataset(i): - dataset = db_models.DATASET() + dataset = models.DATASET() apply_common_attributes(dataset, i) dataset.COMPLETE = randrange(2) dataset.LOCATION = faker.file_path() @@ -576,7 +576,7 @@ def generate(self): @staticmethod def generate_dataset_parameter(i): - dataset_param = db_models.DATASETPARAMETER() + dataset_param = models.DATASETPARAMETER() apply_common_attributes(dataset_param, i) apply_common_parameter_attributes(dataset_param, i) dataset_param.DATASET_ID = randrange(1, DatasetGenerator.amount) @@ -593,7 +593,7 @@ def generate(self): @staticmethod def generate_datafile(i): - datafile = db_models.DATAFILE() + datafile = models.DATAFILE() apply_common_attributes(datafile, i % 19) datafile.CHECKSUM = faker.md5() datafile.DATAFILECREATETIME = datafile.CREATE_TIME @@ -615,7 +615,7 @@ def generate(self): def generate_permissible_string_value(i): - permissible_string_value = db_models.PERMISSIBLESTRINGVALUE() + permissible_string_value = models.PERMISSIBLESTRINGVALUE() apply_common_attributes(permissible_string_value, i) permissible_string_value.VALUE = f"value {i}" permissible_string_value.PARAMETERTYPE_ID = i @@ -633,7 +633,7 @@ def generate(self): @staticmethod def generate_data_collection_parameter(i): - datacollection_parameter = db_models.DATACOLLECTIONPARAMETER() + datacollection_parameter = models.DATACOLLECTIONPARAMETER() apply_common_attributes(datacollection_parameter, i) apply_common_parameter_attributes(datacollection_parameter, i) datacollection_parameter.DATACOLLECTION_ID = i @@ -652,7 +652,7 @@ def generate(self): @staticmethod def generate_sample_parameter(i): - sample_parameter = db_models.SAMPLEPARAMETER() + sample_parameter = models.SAMPLEPARAMETER() apply_common_attributes(sample_parameter, i) apply_common_parameter_attributes(sample_parameter, i) sample_parameter.SAMPLE_ID = i @@ -669,7 +669,7 @@ def generate(self): @staticmethod def generate_datafile_parameter(i): - datafile_param = db_models.DATAFILEPARAMETER() + datafile_param = models.DATAFILEPARAMETER() apply_common_attributes(datafile_param, i) apply_common_parameter_attributes(datafile_param, i) datafile_param.DATAFILE_ID = i