From 8599c5e54c0b9a6ed50fdab8f335cfc504842857 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Thu, 20 Jun 2019 13:06:59 +0100 Subject: [PATCH 01/18] #2: Fix typo --- common/models/db_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/models/db_models.py b/common/models/db_models.py index 2093e859..cc40b023 100644 --- a/common/models/db_models.py +++ b/common/models/db_models.py @@ -13,7 +13,7 @@ class EntityHelper(object): def to_dict(self): """ Turns the columns and values of an entity into a dictionary - :return: dict: dictionay containing the fields and values of an entity + :return: dict: dictionary containing the fields and values of an entity """ dictionary = {} for column in self.__table__.columns: From b243897c27571fb5afa5584561780404f4c9a81d Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Thu, 20 Jun 2019 13:08:02 +0100 Subject: [PATCH 02/18] #2: Add order filtering --- common/database_helpers.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 9b7fa768..992d5f32 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -1,6 +1,6 @@ import datetime -from sqlalchemy import create_engine +from sqlalchemy import create_engine, asc, desc from sqlalchemy.orm import sessionmaker from common.constants import Constants @@ -127,8 +127,15 @@ def update_row_from_id(table, id, new_values): def get_rows_by_filter(table, filters): + """ + Given a list of filters supplied in json format, returns entities that match the filters from the given table + :param table: The table to checked + :param filters: The filters to be applied + :return: A list of the rows returned in dictionary form + """ session = get_icat_db_session() base_query = session.query(table) + print(filters) for filter in filters: if list(filter)[0].lower() == "where": for key in filter: @@ -136,9 +143,15 @@ def get_rows_by_filter(table, filters): for k in where_part: column = getattr(table, k.upper()) base_query = base_query.filter(column.in_([where_part[k]])) - elif list(filter)[0].lower() == "order": - base_query.order() # do something probably not .order + field = filter["order"].split(" ")[0] + direction = filter["order"].split(" ")[1] + if direction.upper() == "ASC": + base_query = base_query.order_by(asc(getattr(table ,field))) + elif direction.upper() == "DESC": + base_query = base_query.order_by(desc(getattr(table ,field))) + else: + raise BadFilterError() elif list(filter)[0].lower() == "skip": for key in filter: skip = filter[key] From 36352dcb3da72d688546c6baf043e2c973a39df5 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Fri, 21 Jun 2019 10:47:29 +0100 Subject: [PATCH 03/18] #2: Allow ordering before limit and vise versa --- common/database_helpers.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 691bee50..3158b468 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -120,9 +120,9 @@ def get_rows_by_filter(table, filters): :param filters: The filters to be applied :return: A list of the rows returned in dictionary form """ + is_limited = False session = get_icat_db_session() base_query = session.query(table) - print(filters) for filter in filters: if list(filter)[0].lower() == "where": for key in filter: @@ -133,12 +133,21 @@ def get_rows_by_filter(table, filters): elif list(filter)[0].lower() == "order": field = filter["order"].split(" ")[0] direction = filter["order"].split(" ")[1] - if direction.upper() == "ASC": - base_query = base_query.order_by(asc(getattr(table ,field))) - elif direction.upper() == "DESC": - base_query = base_query.order_by(desc(getattr(table ,field))) + + if is_limited: + if direction.upper() == "ASC": + base_query = base_query.from_self().order_by(asc(getattr(table, field))) + elif direction.upper() == "DESC": + base_query = base_query.from_self().order_by(desc(getattr(table, field))) + else: + raise BadFilterError() else: - raise BadFilterError() + if direction.upper() == "ASC": + base_query = base_query.order_by(asc(getattr(table, field))) + elif direction.upper() == "DESC": + base_query = base_query.order_by(desc(getattr(table, field))) + else: + raise BadFilterError() elif list(filter)[0].lower() == "skip": for key in filter: skip = filter[key] @@ -146,6 +155,7 @@ def get_rows_by_filter(table, filters): elif list(filter)[0].lower() == "include": base_query.include() # do something probably not .include elif list(filter)[0].lower() == "limit": + is_limited = True for key in filter: limit = filter[key] base_query = base_query.limit(limit) From c45075ae199b83d6e35511a419001a3f4c3260e1 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Fri, 21 Jun 2019 11:01:58 +0100 Subject: [PATCH 04/18] #2: Change how exceptions are logged --- common/database_helpers.py | 14 +++++++------- common/exceptions.py | 16 +++++----------- common/helpers.py | 2 +- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 3158b468..b3133359 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -71,7 +71,7 @@ def get_row_by_id(table, id): session.close() return result session.close() - raise MissingRecordError() + raise MissingRecordError(f" Could not find record in {table.__tablename__} with ID: {id}") def delete_row_by_id(table, id): @@ -90,7 +90,7 @@ def delete_row_by_id(table, id): session.close() return session.close() - raise MissingRecordError() + raise MissingRecordError(f" Could not find record in {table.__tablename__} with ID: {id}") def update_row_from_id(table, id, new_values): @@ -110,7 +110,7 @@ def update_row_from_id(table, id, new_values): session.close() return session.close() - raise MissingRecordError() + raise MissingRecordError(f" Could not find record in {table.__tablename__} with ID: {id}") def get_rows_by_filter(table, filters): @@ -140,14 +140,14 @@ def get_rows_by_filter(table, filters): elif direction.upper() == "DESC": base_query = base_query.from_self().order_by(desc(getattr(table, field))) else: - raise BadFilterError() + raise BadFilterError(f" Bad filter given, filter: {filter}") else: if direction.upper() == "ASC": base_query = base_query.order_by(asc(getattr(table, field))) elif direction.upper() == "DESC": base_query = base_query.order_by(desc(getattr(table, field))) else: - raise BadFilterError() + raise BadFilterError(f" Bad filter given, filter: {filter}") elif list(filter)[0].lower() == "skip": for key in filter: skip = filter[key] @@ -160,7 +160,7 @@ def get_rows_by_filter(table, filters): limit = filter[key] base_query = base_query.limit(limit) else: - raise BadFilterError() + raise BadFilterError(f"Invalid filters provided recieved {filters}") log.info(" Closing DB session") session.close() return list(map(lambda x: x.to_dict(), base_query.all())) @@ -211,6 +211,6 @@ def patch_entities(table, json_list): result = get_row_by_id(table, entity[key]) results.append(result) if len(results) == 0: - raise BadRequestError() + raise BadRequestError(f" Bad request made, request: {json_list}") return results diff --git a/common/exceptions.py b/common/exceptions.py index 29741848..56dc89ee 100644 --- a/common/exceptions.py +++ b/common/exceptions.py @@ -4,26 +4,20 @@ class ApiError(Exception): - def __init__(self): - log.info(" ApiError(): An error has been raised.") + pass class MissingRecordError(ApiError): - def __init__(self): - log.info(" MissingRecordError(): Record not found, DB session Closed") - + pass class BadFilterError(ApiError): - def __init__(self): - log.info(" BadFilterError(): Bad filter supplied") + pass class AuthenticationError(ApiError): - def __init__(self): - log.info(" AuthenticationError(): Error authenticating consumer") + pass class BadRequestError(ApiError): - def __init__(self): - log.info(" BadRequestError(): Bad request by Consumer") + pass diff --git a/common/helpers.py b/common/helpers.py index 079cb785..dab96995 100644 --- a/common/helpers.py +++ b/common/helpers.py @@ -87,7 +87,7 @@ def get_session_id_from_auth_header(): if auth_header == "": return "" if len(auth_header) != 2 or auth_header[0] != "Bearer": - raise AuthenticationError() + raise AuthenticationError(f" Could not authenticate consumer with auth header {auth_header}") return auth_header[1] From 8f776cd296117473c92c89d6db8b727411a7ad6d Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 24 Jun 2019 10:47:27 +0100 Subject: [PATCH 05/18] #2: Add limit and order filters --- common/database_helpers.py | 16 ++++++++++------ common/helpers.py | 2 +- test/test_base/base_rest_test.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index b3133359..04a0dcf6 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -129,11 +129,13 @@ def get_rows_by_filter(table, filters): where_part = filter[key] for k in where_part: column = getattr(table, k.upper()) - base_query = base_query.filter(column.in_([where_part[k]])) - elif list(filter)[0].lower() == "order": - field = filter["order"].split(" ")[0] - direction = filter["order"].split(" ")[1] + base_query = base_query.filter(column.in_([where_part[k]]), column.in_([where_part[k]])) + elif list(filter)[0].lower() == "order": + for key in filter: + field = filter[key].split(" ")[0] + direction = filter[key].split(" ")[1] + # Limit then order, or order then limit if is_limited: if direction.upper() == "ASC": base_query = base_query.from_self().order_by(asc(getattr(table, field))) @@ -148,19 +150,21 @@ def get_rows_by_filter(table, filters): base_query = base_query.order_by(desc(getattr(table, field))) else: raise BadFilterError(f" Bad filter given, filter: {filter}") + elif list(filter)[0].lower() == "skip": for key in filter: skip = filter[key] base_query = base_query.offset(skip) - elif list(filter)[0].lower() == "include": - base_query.include() # do something probably not .include + elif list(filter)[0].lower() == "limit": is_limited = True for key in filter: limit = filter[key] base_query = base_query.limit(limit) + else: raise BadFilterError(f"Invalid filters provided recieved {filters}") + log.info(" Closing DB session") session.close() return list(map(lambda x: x.to_dict(), base_query.all())) diff --git a/common/helpers.py b/common/helpers.py index dab96995..47062dc1 100644 --- a/common/helpers.py +++ b/common/helpers.py @@ -33,7 +33,7 @@ def wrapper_requires_session(*args, **kwargs): log.info(" Consumer authenticated") return method(*args, **kwargs) else: - log.info(" Closing DB session") + log.info(" Could not authenticate consumer, closing DB session") session.close() return "Forbidden", 403 except AuthenticationError: diff --git a/test/test_base/base_rest_test.py b/test/test_base/base_rest_test.py index c839eff3..20becf5a 100644 --- a/test/test_base/base_rest_test.py +++ b/test/test_base/base_rest_test.py @@ -28,7 +28,7 @@ def tearDown(self): def expect_status_code(self, expected_status_code, response): """ - Asserts whethere the returned status code is equal to the expected + Asserts whether the returned status code is equal to the expected :param expected_status_code: int: The status code that is expected :param response: The response to be checked """ From bf00f23db7df0371e86868c1fbee0acfb01ce448 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 1 Jul 2019 15:24:18 +0100 Subject: [PATCH 06/18] #2: Update backref on db models --- common/models/db_models.py | 109 ++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/common/models/db_models.py b/common/models/db_models.py index cc40b023..22a86c2f 100644 --- a/common/models/db_models.py +++ b/common/models/db_models.py @@ -44,7 +44,7 @@ class APPLICATION(Base, EntityHelper): VERSION = Column(String(255), nullable=False) FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) - FACILITY = relationship('FACILITY', primaryjoin='APPLICATION.FACILITY_ID == FACILITY.ID', backref='applications') + FACILITY = relationship('FACILITY', primaryjoin='APPLICATION.FACILITY_ID == FACILITY.ID', backref='APPLICATION') class FACILITY(Base, EntityHelper): @@ -89,9 +89,9 @@ class DATACOLLECTIONDATAFILE(Base, EntityHelper): DATACOLLECTION = relationship('DATACOLLECTION', primaryjoin='DATACOLLECTIONDATAFILE.DATACOLLECTION_ID == DATACOLLECTION.ID', - backref='datacollectiondatafiles') + backref='DATACOLLECTIONDATAFILE') DATAFILE = relationship('DATAFILE', primaryjoin='DATACOLLECTIONDATAFILE.DATAFILE_ID == DATAFILE.ID', - backref='datacollectiondatafiles') + backref='DATACOLLECTIONDATAFILE') class DATACOLLECTIONDATASET(Base, EntityHelper): @@ -110,9 +110,9 @@ class DATACOLLECTIONDATASET(Base, EntityHelper): DATACOLLECTION = relationship('DATACOLLECTION', primaryjoin='DATACOLLECTIONDATASET.DATACOLLECTION_ID == DATACOLLECTION.ID', - backref='datacollectiondatasets') + backref='DATACOLLECTIONDATASET') DATASET = relationship('DATASET', primaryjoin='DATACOLLECTIONDATASET.DATASET_ID == DATASET.ID', - backref='datacollectiondatasets') + backref='DATACOLLECTIONDATASET') class DATACOLLECTIONPARAMETER(Base, EntityHelper): @@ -137,10 +137,10 @@ class DATACOLLECTIONPARAMETER(Base, EntityHelper): DATACOLLECTION = relationship('DATACOLLECTION', primaryjoin='DATACOLLECTIONPARAMETER.DATACOLLECTION_ID == DATACOLLECTION.ID', - backref='datacollectionparameters') + backref='DATACOLLECTIONPARAMETER') PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='DATACOLLECTIONPARAMETER.PARAMETER_TYPE_ID == PARAMETERTYPE.ID', - backref='datacollectionparameters') + backref='DATACOLLECTIONPARAMETER') class DATAFILE(Base, EntityHelper): @@ -166,8 +166,8 @@ class DATAFILE(Base, EntityHelper): DATASET_ID = Column(ForeignKey('DATASET.ID'), nullable=False) DATAFILEFORMAT = relationship('DATAFILEFORMAT', primaryjoin='DATAFILE.DATAFILEFORMAT_ID == DATAFILEFORMAT.ID', - backref='datafiles') - DATASET = relationship('DATASET', primaryjoin='DATAFILE.DATASET_ID == DATASET.ID', backref='datafiles') + backref='DATAFILE') + DATASET = relationship('DATASET', primaryjoin='DATAFILE.DATASET_ID == DATASET.ID', backref='DATAFILE') class DATAFILEFORMAT(Base, EntityHelper): @@ -188,7 +188,7 @@ class DATAFILEFORMAT(Base, EntityHelper): FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) FACILITY = relationship('FACILITY', primaryjoin='DATAFILEFORMAT.FACILITY_ID == FACILITY.ID', - backref='datafileformats') + backref='DATAFILEFORMAT') class DATAFILEPARAMETER(Base, EntityHelper): @@ -212,9 +212,9 @@ class DATAFILEPARAMETER(Base, EntityHelper): PARAMETER_TYPE_ID = Column(ForeignKey('PARAMETERTYPE.ID'), nullable=False, index=True) DATAFILE = relationship('DATAFILE', primaryjoin='DATAFILEPARAMETER.DATAFILE_ID == DATAFILE.ID', - backref='datafileparameters') + backref='DATAFILEPARAMETER') PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='DATAFILEPARAMETER.PARAMETER_TYPE_ID == PARAMETERTYPE.ID', - backref='datafileparameters') + backref='DATAFILEPARAMETER') class DATASET(Base, EntityHelper): @@ -240,9 +240,9 @@ class DATASET(Base, EntityHelper): TYPE_ID = Column(ForeignKey('DATASETTYPE.ID'), nullable=False, index=True) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='DATASET.INVESTIGATION_ID == INVESTIGATION.ID', - backref='datasets') - SAMPLE = relationship('SAMPLE', primaryjoin='DATASET.SAMPLE_ID == SAMPLE.ID', backref='datasets') - DATASETTYPE = relationship('DATASETTYPE', primaryjoin='DATASET.TYPE_ID == DATASETTYPE.ID', backref='datasets') + backref='DATASET') + SAMPLE = relationship('SAMPLE', primaryjoin='DATASET.SAMPLE_ID == SAMPLE.ID', backref='DATASET') + DATASETTYPE = relationship('DATASETTYPE', primaryjoin='DATASET.TYPE_ID == DATASETTYPE.ID', backref='DATASET') class DATASETPARAMETER(Base, EntityHelper): @@ -266,9 +266,9 @@ class DATASETPARAMETER(Base, EntityHelper): PARAMETER_TYPE_ID = Column(ForeignKey('PARAMETERTYPE.ID'), nullable=False, index=True) DATASET = relationship('DATASET', primaryjoin='DATASETPARAMETER.DATASET_ID == DATASET.ID', - backref='datasetparameters') + backref='DATASETPARAMETER') PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='DATASETPARAMETER.PARAMETER_TYPE_ID == PARAMETERTYPE.ID', - backref='datasetparameters') + backref='DATASETPARAMETER') class DATASETTYPE(Base, EntityHelper): @@ -286,7 +286,7 @@ class DATASETTYPE(Base, EntityHelper): NAME = Column(String(255), nullable=False) FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) - FACILITY = relationship('FACILITY', primaryjoin='DATASETTYPE.FACILITY_ID == FACILITY.ID', backref='datasettypes') + FACILITY = relationship('FACILITY', primaryjoin='DATASETTYPE.FACILITY_ID == FACILITY.ID', backref='DATASETTYPE') class FACILITYCYCLE(Base, EntityHelper): @@ -307,7 +307,7 @@ class FACILITYCYCLE(Base, EntityHelper): FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) FACILITY = relationship('FACILITY', primaryjoin='FACILITYCYCLE.FACILITY_ID == FACILITY.ID', - backref='facilitycycles') + backref='FACILITYCYCLE') class GROUPING(Base, EntityHelper): @@ -339,7 +339,7 @@ class INSTRUMENT(Base, EntityHelper): URL = Column(String(255)) FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) - FACILITY = relationship('FACILITY', primaryjoin='INSTRUMENT.FACILITY_ID == FACILITY.ID', backref='instruments') + FACILITY = relationship('FACILITY', primaryjoin='INSTRUMENT.FACILITY_ID == FACILITY.ID', backref='INSTRUMENT') class INSTRUMENTSCIENTIST(Base, EntityHelper): @@ -357,8 +357,8 @@ class INSTRUMENTSCIENTIST(Base, EntityHelper): USER_ID = Column(ForeignKey('USER_.ID'), nullable=False) INSTRUMENT = relationship('INSTRUMENT', primaryjoin='INSTRUMENTSCIENTIST.INSTRUMENT_ID == INSTRUMENT.ID', - backref='instrumentscientists') - USER_ = relationship('USER', primaryjoin='INSTRUMENTSCIENTIST.USER_ID == USER.ID', backref='instrumentscientists') + backref='INSTRUMENTSCIENTIST') + USER_ = relationship('USER', primaryjoin='INSTRUMENTSCIENTIST.USER_ID == USER.ID', backref='INSTRUMENTSCIENTIST') class INVESTIGATION(Base, EntityHelper): @@ -384,9 +384,9 @@ class INVESTIGATION(Base, EntityHelper): TYPE_ID = Column(ForeignKey('INVESTIGATIONTYPE.ID'), nullable=False, index=True) FACILITY = relationship('FACILITY', primaryjoin='INVESTIGATION.FACILITY_ID == FACILITY.ID', - backref='investigations') + backref='INVESTIGATION') INVESTIGATIONTYPE = relationship('INVESTIGATIONTYPE', primaryjoin='INVESTIGATION.TYPE_ID == INVESTIGATIONTYPE.ID', - backref='investigations') + backref='INVESTIGATION') class INVESTIGATIONGROUP(Base, EntityHelper): @@ -405,9 +405,9 @@ class INVESTIGATIONGROUP(Base, EntityHelper): INVESTIGATION_ID = Column(ForeignKey('INVESTIGATION.ID'), nullable=False, index=True) GROUPING = relationship('GROUPING', primaryjoin='INVESTIGATIONGROUP.GROUP_ID == GROUPING.ID', - backref='investigationgroups') + backref='INVESTIGATIONGROUP') INVESTIGATION = relationship('INVESTIGATION', primaryjoin='INVESTIGATIONGROUP.INVESTIGATION_ID == INVESTIGATION.ID', - backref='investigationgroups') + backref='INVESTIGATIONGROUP') class INVESTIGATIONINSTRUMENT(Base, EntityHelper): @@ -425,10 +425,10 @@ class INVESTIGATIONINSTRUMENT(Base, EntityHelper): INVESTIGATION_ID = Column(ForeignKey('INVESTIGATION.ID'), nullable=False) INSTRUMENT = relationship('INSTRUMENT', primaryjoin='INVESTIGATIONINSTRUMENT.INSTRUMENT_ID == INSTRUMENT.ID', - backref='investigationinstruments') + backref='INVESTIGATIONINSTRUMENT') INVESTIGATION = relationship('INVESTIGATION', primaryjoin='INVESTIGATIONINSTRUMENT.INVESTIGATION_ID == INVESTIGATION.ID', - backref='investigationinstruments') + backref='INVESTIGATIONINSTRUMENT') class INVESTIGATIONPARAMETER(Base, EntityHelper): @@ -453,10 +453,10 @@ class INVESTIGATIONPARAMETER(Base, EntityHelper): INVESTIGATION = relationship('INVESTIGATION', primaryjoin='INVESTIGATIONPARAMETER.INVESTIGATION_ID == INVESTIGATION.ID', - backref='investigationparameters') + backref='INVESTIGATIONPARAMETER') PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='INVESTIGATIONPARAMETER.PARAMETER_TYPE_ID == PARAMETERTYPE.ID', - backref='investigationparameters') + backref='INVESTIGATIONPARAMETER') class INVESTIGATIONTYPE(Base, EntityHelper): @@ -475,7 +475,7 @@ class INVESTIGATIONTYPE(Base, EntityHelper): FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False, index=True) FACILITY = relationship('FACILITY', primaryjoin='INVESTIGATIONTYPE.FACILITY_ID == FACILITY.ID', - backref='investigationtypes') + backref='INVESTIGATIONTYPE') class INVESTIGATIONUSER(Base, EntityHelper): @@ -494,8 +494,8 @@ class INVESTIGATIONUSER(Base, EntityHelper): USER_ID = Column(ForeignKey('USER_.ID'), nullable=False) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='INVESTIGATIONUSER.INVESTIGATION_ID == INVESTIGATION.ID', - backref='investigationusers') - USER_ = relationship('USER', primaryjoin='INVESTIGATIONUSER.USER_ID == USER.ID', backref='investigationusers') + backref='INVESTIGATIONUSER') + USER_ = relationship('USER', primaryjoin='INVESTIGATIONUSER.USER_ID == USER.ID', backref='INVESTIGATIONUSER') class JOB(Base, EntityHelper): @@ -511,11 +511,10 @@ class JOB(Base, EntityHelper): INPUTDATACOLLECTION_ID = Column(ForeignKey('DATACOLLECTION.ID'), index=True) OUTPUTDATACOLLECTION_ID = Column(ForeignKey('DATACOLLECTION.ID'), index=True) - APPLICATION = relationship('APPLICATION', primaryjoin='JOB.APPLICATION_ID == APPLICATION.ID', backref='jobs') + APPLICATION = relationship('APPLICATION', primaryjoin='JOB.APPLICATION_ID == APPLICATION.ID', backref='JOB') DATACOLLECTION = relationship('DATACOLLECTION', primaryjoin='JOB.INPUTDATACOLLECTION_ID == DATACOLLECTION.ID', - backref='datacollection_jobs') - DATACOLLECTION1 = relationship('DATACOLLECTION', primaryjoin='JOB.OUTPUTDATACOLLECTION_ID == DATACOLLECTION.ID', - backref='datacollection_jobs_0') + backref='JOB') + class KEYWORD(Base, EntityHelper): @@ -533,7 +532,7 @@ class KEYWORD(Base, EntityHelper): INVESTIGATION_ID = Column(ForeignKey('INVESTIGATION.ID'), nullable=False, index=True) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='KEYWORD.INVESTIGATION_ID == INVESTIGATION.ID', - backref='keywords') + backref='KEYWORD') class PARAMETERTYPE(Base, EntityHelper): @@ -564,7 +563,7 @@ class PARAMETERTYPE(Base, EntityHelper): FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) FACILITY = relationship('FACILITY', primaryjoin='PARAMETERTYPE.FACILITY_ID == FACILITY.ID', - backref='parametertypes') + backref='PARAMETERTYPE') class PERMISSIBLESTRINGVALUE(Base, EntityHelper): @@ -583,7 +582,7 @@ class PERMISSIBLESTRINGVALUE(Base, EntityHelper): PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='PERMISSIBLESTRINGVALUE.PARAMETERTYPE_ID == PARAMETERTYPE.ID', - backref='permissiblestringvalues') + backref='PERMISSIBLESTRINGVALUE') class PUBLICATION(Base, EntityHelper): @@ -602,7 +601,7 @@ class PUBLICATION(Base, EntityHelper): INVESTIGATION_ID = Column(ForeignKey('INVESTIGATION.ID'), nullable=False, index=True) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='PUBLICATION.INVESTIGATION_ID == INVESTIGATION.ID', - backref='publications') + backref='PUBLICATION') class PUBLICSTEP(Base, EntityHelper): @@ -636,9 +635,7 @@ class RELATEDDATAFILE(Base, EntityHelper): SOURCE_DATAFILE_ID = Column(ForeignKey('DATAFILE.ID'), nullable=False) DATAFILE = relationship('DATAFILE', primaryjoin='RELATEDDATAFILE.DEST_DATAFILE_ID == DATAFILE.ID', - backref='datafile_relateddatafiles') - DATAFILE1 = relationship('DATAFILE', primaryjoin='RELATEDDATAFILE.SOURCE_DATAFILE_ID == DATAFILE.ID', - backref='datafile_relateddatafiles_0') + backref='RELATEDDATAFILE') class RULE(Base, EntityHelper): @@ -663,7 +660,7 @@ class RULE(Base, EntityHelper): WHAT = Column(String(1024), nullable=False) GROUPING_ID = Column(ForeignKey('GROUPING.ID'), index=True) - GROUPING = relationship('GROUPING', primaryjoin='RULE.GROUPING_ID == GROUPING.ID', backref='rules') + GROUPING = relationship('GROUPING', primaryjoin='RULE.GROUPING_ID == GROUPING.ID', backref='RULE') class SAMPLE(Base, EntityHelper): @@ -682,8 +679,8 @@ class SAMPLE(Base, EntityHelper): SAMPLETYPE_ID = Column(ForeignKey('SAMPLETYPE.ID'), index=True) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='SAMPLE.INVESTIGATION_ID == INVESTIGATION.ID', - backref='samples') - SAMPLETYPE = relationship('SAMPLETYPE', primaryjoin='SAMPLE.SAMPLETYPE_ID == SAMPLETYPE.ID', backref='samples') + backref='SAMPLE') + SAMPLETYPE = relationship('SAMPLETYPE', primaryjoin='SAMPLE.SAMPLETYPE_ID == SAMPLETYPE.ID', backref='SAMPLE') class SAMPLEPARAMETER(Base, EntityHelper): @@ -707,8 +704,8 @@ class SAMPLEPARAMETER(Base, EntityHelper): PARAMETER_TYPE_ID = Column(ForeignKey('PARAMETERTYPE.ID'), nullable=False, index=True) PARAMETERTYPE = relationship('PARAMETERTYPE', primaryjoin='SAMPLEPARAMETER.PARAMETER_TYPE_ID == PARAMETERTYPE.ID', - backref='sampleparameters') - SAMPLE = relationship('SAMPLE', primaryjoin='SAMPLEPARAMETER.SAMPLE_ID == SAMPLE.ID', backref='sampleparameters') + backref='SAMPLEPARAMETER') + SAMPLE = relationship('SAMPLE', primaryjoin='SAMPLEPARAMETER.SAMPLE_ID == SAMPLE.ID', backref='SAMPLEPARAMETER') class SESSION(Base, EntityHelper): @@ -736,7 +733,7 @@ class SHIFT(Base, EntityHelper): INVESTIGATION_ID = Column(ForeignKey('INVESTIGATION.ID'), nullable=False) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='SHIFT.INVESTIGATION_ID == INVESTIGATION.ID', - backref='shifts') + backref='SHIFT') class USER(Base, EntityHelper): @@ -767,8 +764,8 @@ class USERGROUP(Base, EntityHelper): GROUP_ID = Column(ForeignKey('GROUPING.ID'), nullable=False, index=True) USER_ID = Column(ForeignKey('USER_.ID'), nullable=False) - GROUPING = relationship('GROUPING', primaryjoin='USERGROUP.GROUP_ID == GROUPING.ID', backref='usergroups') - USER_ = relationship('USER', primaryjoin='USERGROUP.USER_ID == USER.ID', backref='usergroups') + GROUPING = relationship('GROUPING', primaryjoin='USERGROUP.GROUP_ID == GROUPING.ID', backref='USERGROUP') + USER_ = relationship('USER', primaryjoin='USERGROUP.USER_ID == USER.ID', backref='USERGROUP') class STUDYINVESTIGATION(Base, EntityHelper): @@ -786,8 +783,8 @@ class STUDYINVESTIGATION(Base, EntityHelper): STUDY_ID = Column(ForeignKey('STUDY.ID'), nullable=False) INVESTIGATION = relationship('INVESTIGATION', primaryjoin='STUDYINVESTIGATION.INVESTIGATION_ID == INVESTIGATION.ID', - backref='studyinvestigations') - STUDY = relationship('STUDY', primaryjoin='STUDYINVESTIGATION.STUDY_ID == STUDY.ID', backref='studyinvestigations') + backref='STUDYINVESTIGATION') + STUDY = relationship('STUDY', primaryjoin='STUDYINVESTIGATION.STUDY_ID == STUDY.ID', backref='STUDYINVESTIGATION') class STUDY(Base, EntityHelper): @@ -804,7 +801,7 @@ class STUDY(Base, EntityHelper): STATUS = Column(Integer) USER_ID = Column(ForeignKey('USER_.ID'), index=True) - USER_ = relationship('USER', primaryjoin='STUDY.USER_ID == USER.ID', backref='studies') + USER_ = relationship('USER', primaryjoin='STUDY.USER_ID == USER.ID', backref='STUDY') class SAMPLETYPE(Base, EntityHelper): @@ -823,4 +820,4 @@ class SAMPLETYPE(Base, EntityHelper): SAFETYINFORMATION = Column(String(4000)) FACILITY_ID = Column(ForeignKey('FACILITY.ID'), nullable=False) - FACILITY = relationship('FACILITY', primaryjoin='SAMPLETYPE.FACILITY_ID == FACILITY.ID', backref='sampletypes') + FACILITY = relationship('FACILITY', primaryjoin='SAMPLETYPE.FACILITY_ID == FACILITY.ID', backref='SAMPLETYPE') From 21a6e5f147304be8c0a236f0e3e16dae4d812574 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Tue, 2 Jul 2019 07:56:20 +0100 Subject: [PATCH 07/18] #2: Allow a single include --- common/database_helpers.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 04a0dcf6..93d2aa0b 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -3,6 +3,7 @@ from sqlalchemy import create_engine, asc, desc from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.collections import InstrumentedList from common.constants import Constants from common.exceptions import MissingRecordError, BadFilterError, BadRequestError @@ -117,12 +118,13 @@ def get_rows_by_filter(table, filters): """ Given a list of filters supplied in json format, returns entities that match the filters from the given table :param table: The table to checked - :param filters: The filters to be applied + :param filters: The list of filters to be applied :return: A list of the rows returned in dictionary form """ is_limited = False session = get_icat_db_session() base_query = session.query(table) + includes_relation = False for filter in filters: if list(filter)[0].lower() == "where": for key in filter: @@ -161,13 +163,34 @@ def get_rows_by_filter(table, filters): for key in filter: limit = filter[key] base_query = base_query.limit(limit) + elif list(filter)[0].lower() == "include": + includes_relation = True else: raise BadFilterError(f"Invalid filters provided recieved {filters}") + results = base_query.all() + if includes_relation: + included_relationships = [] + for filter in filters: + if list(filter)[0] == "include": + included_relationships.append(filter["include"]) + included_results = [] + for row in results: + for relation in included_relationships: + # Here we check if the included result returns a list of children and if so iterate through them and + # add them to the results. + if isinstance(getattr(row, relation.upper()),InstrumentedList): + for i in getattr(row, relation.upper()): + included_results.append(i) + else: + included_results.append(getattr(row, relation.upper())) + results.extend(included_results) + + log.info(" Closing DB session") session.close() - return list(map(lambda x: x.to_dict(), base_query.all())) + return list(map(lambda x: x.to_dict(), results)) def get_filtered_row_count(table, filters): From e7485a6e88e26568f4a48372eb406f1f8e154bb7 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Tue, 2 Jul 2019 08:04:29 +0100 Subject: [PATCH 08/18] #2: Allow including multiple relations --- common/database_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 93d2aa0b..5bea1d7f 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -174,13 +174,16 @@ def get_rows_by_filter(table, filters): included_relationships = [] for filter in filters: if list(filter)[0] == "include": + if type(filter["include"]) == str: included_relationships.append(filter["include"]) + if type(filter["include"]) == list: + included_relationships.extend(filter["include"]) included_results = [] for row in results: for relation in included_relationships: # Here we check if the included result returns a list of children and if so iterate through them and # add them to the results. - if isinstance(getattr(row, relation.upper()),InstrumentedList): + if isinstance(getattr(row, relation.upper()), InstrumentedList): for i in getattr(row, relation.upper()): included_results.append(i) else: From d0a23134e6c757ab0ed5ed162023339ffb1225d5 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Tue, 2 Jul 2019 08:29:14 +0100 Subject: [PATCH 09/18] #2: Allow includes from dictionary --- common/database_helpers.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 5bea1d7f..eadfd23e 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -167,18 +167,27 @@ def get_rows_by_filter(table, filters): includes_relation = True else: - raise BadFilterError(f"Invalid filters provided recieved {filters}") + raise BadFilterError(f"Invalid filters provided received {filters}") results = base_query.all() if includes_relation: included_relationships = [] + included_included_relationships = [] for filter in filters: if list(filter)[0] == "include": if type(filter["include"]) == str: included_relationships.append(filter["include"]) - if type(filter["include"]) == list: + elif type(filter["include"]) == list: included_relationships.extend(filter["include"]) + elif type(filter["include"]) == dict: + for key in filter["include"]: + included_relationships.append(key) + included_included_relationships.append(filter["include"][key]) + else: + raise BadFilterError(f" Invalid format of included relationships") + included_results = [] + included_included_results = [] for row in results: for relation in included_relationships: # Here we check if the included result returns a list of children and if so iterate through them and @@ -188,9 +197,15 @@ def get_rows_by_filter(table, filters): included_results.append(i) else: included_results.append(getattr(row, relation.upper())) + for row in included_results: + for relation in included_included_relationships: + if isinstance(getattr(row, relation.upper()), InstrumentedList): + for i in getattr(row, relation.upper()): + included_included_results.append(i) + else: + included_included_results.append(getattr(row, relation.upper())) results.extend(included_results) - - + results.extend(included_included_results) log.info(" Closing DB session") session.close() return list(map(lambda x: x.to_dict(), results)) From e305449f06ffe1d169f4ba2db1841ebd2cc6550b Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Tue, 2 Jul 2019 09:15:38 +0100 Subject: [PATCH 10/18] #2: Pull out including to func for readability --- common/database_helpers.py | 39 +++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index eadfd23e..e12fb61f 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -170,21 +170,36 @@ def get_rows_by_filter(table, filters): raise BadFilterError(f"Invalid filters provided received {filters}") results = base_query.all() + # check if include was provided, then add included results if includes_relation: - included_relationships = [] - included_included_relationships = [] for filter in filters: if list(filter)[0] == "include": - if type(filter["include"]) == str: - included_relationships.append(filter["include"]) - elif type(filter["include"]) == list: - included_relationships.extend(filter["include"]) - elif type(filter["include"]) == dict: - for key in filter["include"]: + results = get_related_entities(filter["include"], results) + log.info(" Closing DB session") + session.close() + return list(map(lambda x: x.to_dict(), results)) + + +def get_related_entities(include_filters, results): + """ + Given a set of results from a query and an include filter of the form str, dict or list append to the results + related entities + :param include_filters: the include filter: str, dict or list + :param results: list of rows from a query + :return: updated list of rows with related entities + """ + included_relationships = [] + included_included_relationships = [] + if type(include_filters) == str: + included_relationships.append(include_filters) + elif type(include_filters) == list: + included_relationships.extend(include_filters) + elif type(include_filters) == dict: + for key in include_filters: included_relationships.append(key) - included_included_relationships.append(filter["include"][key]) + included_included_relationships.append(include_filters[key]) else: - raise BadFilterError(f" Invalid format of included relationships") + raise BadFilterError(" Invalid format of included relationships") included_results = [] included_included_results = [] @@ -206,9 +221,7 @@ def get_rows_by_filter(table, filters): included_included_results.append(getattr(row, relation.upper())) results.extend(included_results) results.extend(included_included_results) - log.info(" Closing DB session") - session.close() - return list(map(lambda x: x.to_dict(), results)) + return results def get_filtered_row_count(table, filters): From e72561d571d396d90f392275b0f05a90673fc818 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Tue, 2 Jul 2019 09:20:34 +0100 Subject: [PATCH 11/18] #2: Change name --- common/database_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index e12fb61f..ff5b93a7 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -212,13 +212,13 @@ def get_related_entities(include_filters, results): included_results.append(i) else: included_results.append(getattr(row, relation.upper())) - for row in included_results: + for included_row in included_results: for relation in included_included_relationships: - if isinstance(getattr(row, relation.upper()), InstrumentedList): - for i in getattr(row, relation.upper()): + if isinstance(getattr(included_row, relation.upper()), InstrumentedList): + for i in getattr(included_row, relation.upper()): included_included_results.append(i) else: - included_included_results.append(getattr(row, relation.upper())) + included_included_results.append(getattr(included_row, relation.upper())) results.extend(included_results) results.extend(included_included_results) return results From af5c0c28191d0cc06ddbf6b67c12e9b60e9d6c6a Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Thu, 18 Jul 2019 13:05:34 +0100 Subject: [PATCH 12/18] #2: Rename shadow named vars --- common/database_helpers.py | 92 +++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index ff5b93a7..b3f3ec58 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -125,18 +125,18 @@ def get_rows_by_filter(table, filters): session = get_icat_db_session() base_query = session.query(table) includes_relation = False - for filter in filters: - if list(filter)[0].lower() == "where": - for key in filter: - where_part = filter[key] + for query_filter in filters: + if list(query_filter)[0].lower() == "where": + for key in query_filter: + where_part = query_filter[key] for k in where_part: column = getattr(table, k.upper()) base_query = base_query.filter(column.in_([where_part[k]]), column.in_([where_part[k]])) - elif list(filter)[0].lower() == "order": - for key in filter: - field = filter[key].split(" ")[0] - direction = filter[key].split(" ")[1] + elif list(query_filter)[0].lower() == "order": + for key in query_filter: + field = query_filter[key].split(" ")[0] + direction = query_filter[key].split(" ")[1] # Limit then order, or order then limit if is_limited: if direction.upper() == "ASC": @@ -144,26 +144,26 @@ def get_rows_by_filter(table, filters): elif direction.upper() == "DESC": base_query = base_query.from_self().order_by(desc(getattr(table, field))) else: - raise BadFilterError(f" Bad filter given, filter: {filter}") + raise BadFilterError(f" Bad filter given, filter: {query_filter}") else: if direction.upper() == "ASC": base_query = base_query.order_by(asc(getattr(table, field))) elif direction.upper() == "DESC": base_query = base_query.order_by(desc(getattr(table, field))) else: - raise BadFilterError(f" Bad filter given, filter: {filter}") + raise BadFilterError(f" Bad filter given, filter: {query_filter}") - elif list(filter)[0].lower() == "skip": - for key in filter: - skip = filter[key] + elif list(query_filter)[0].lower() == "skip": + for key in query_filter: + skip = query_filter[key] base_query = base_query.offset(skip) - elif list(filter)[0].lower() == "limit": + elif list(query_filter)[0].lower() == "limit": is_limited = True - for key in filter: - limit = filter[key] - base_query = base_query.limit(limit) - elif list(filter)[0].lower() == "include": + for key in query_filter: + query_limit = query_filter[key] + base_query = base_query.limit(query_limit) + elif list(query_filter)[0].lower() == "include": includes_relation = True else: @@ -172,9 +172,9 @@ def get_rows_by_filter(table, filters): results = base_query.all() # check if include was provided, then add included results if includes_relation: - for filter in filters: - if list(filter)[0] == "include": - results = get_related_entities(filter["include"], results) + for query_filter in filters: + if list(query_filter)[0] == "include": + results = get_related_entities(query_filter["include"], results) log.info(" Closing DB session") session.close() return list(map(lambda x: x.to_dict(), results)) @@ -188,40 +188,40 @@ def get_related_entities(include_filters, results): :param results: list of rows from a query :return: updated list of rows with related entities """ - included_relationships = [] - included_included_relationships = [] + included_relationships = [] + included_included_relationships = [] if type(include_filters) == str: included_relationships.append(include_filters) elif type(include_filters) == list: included_relationships.extend(include_filters) elif type(include_filters) == dict: for key in include_filters: - included_relationships.append(key) + included_relationships.append(key) included_included_relationships.append(include_filters[key]) - else: + else: raise BadFilterError(" Invalid format of included relationships") - included_results = [] - included_included_results = [] - for row in results: - for relation in included_relationships: - # Here we check if the included result returns a list of children and if so iterate through them and - # add them to the results. - if isinstance(getattr(row, relation.upper()), InstrumentedList): - for i in getattr(row, relation.upper()): - included_results.append(i) - else: - included_results.append(getattr(row, relation.upper())) - for included_row in included_results: - for relation in included_included_relationships: - if isinstance(getattr(included_row, relation.upper()), InstrumentedList): - for i in getattr(included_row, relation.upper()): - included_included_results.append(i) - else: - included_included_results.append(getattr(included_row, relation.upper())) - results.extend(included_results) - results.extend(included_included_results) - return results + included_results = [] + included_included_results = [] + for row in results: + for relation in included_relationships: + # Here we check if the included result returns a list of children and if so iterate through them and + # add them to the results. + if isinstance(getattr(row, relation.upper()), InstrumentedList): + for i in getattr(row, relation.upper()): + included_results.append(i) + else: + included_results.append(getattr(row, relation.upper())) + for included_row in included_results: + for relation in included_included_relationships: + if isinstance(getattr(included_row, relation.upper()), InstrumentedList): + for i in getattr(included_row, relation.upper()): + included_included_results.append(i) + else: + included_included_results.append(getattr(included_row, relation.upper())) + results.extend(included_results) + results.extend(included_included_results) + return results def get_filtered_row_count(table, filters): From be24106e87c0e778114f7154e0b7b4ae62f3cf07 Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 08:02:00 +0100 Subject: [PATCH 13/18] #2: Remove function --- common/database_helpers.py | 44 -------------------------------------- 1 file changed, 44 deletions(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index b3f3ec58..9bb6b6f0 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -180,50 +180,6 @@ def get_rows_by_filter(table, filters): return list(map(lambda x: x.to_dict(), results)) -def get_related_entities(include_filters, results): - """ - Given a set of results from a query and an include filter of the form str, dict or list append to the results - related entities - :param include_filters: the include filter: str, dict or list - :param results: list of rows from a query - :return: updated list of rows with related entities - """ - included_relationships = [] - included_included_relationships = [] - if type(include_filters) == str: - included_relationships.append(include_filters) - elif type(include_filters) == list: - included_relationships.extend(include_filters) - elif type(include_filters) == dict: - for key in include_filters: - included_relationships.append(key) - included_included_relationships.append(include_filters[key]) - else: - raise BadFilterError(" Invalid format of included relationships") - - included_results = [] - included_included_results = [] - for row in results: - for relation in included_relationships: - # Here we check if the included result returns a list of children and if so iterate through them and - # add them to the results. - if isinstance(getattr(row, relation.upper()), InstrumentedList): - for i in getattr(row, relation.upper()): - included_results.append(i) - else: - included_results.append(getattr(row, relation.upper())) - for included_row in included_results: - for relation in included_included_relationships: - if isinstance(getattr(included_row, relation.upper()), InstrumentedList): - for i in getattr(included_row, relation.upper()): - included_included_results.append(i) - else: - included_included_results.append(getattr(included_row, relation.upper())) - results.extend(included_results) - results.extend(included_included_results) - return results - - def get_filtered_row_count(table, filters): """ returns the count of the rows that match a given filter in a given table From 875fe035e90b73d9adf7e243a9e84ecc5a94493a Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 08:03:02 +0100 Subject: [PATCH 14/18] #2: Add nested dict method --- common/models/db_models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/common/models/db_models.py b/common/models/db_models.py index 22a86c2f..6b92bf89 100644 --- a/common/models/db_models.py +++ b/common/models/db_models.py @@ -1,6 +1,7 @@ from sqlalchemy import Index, Column, BigInteger, String, DateTime, ForeignKey, Integer, Float, FetchedValue from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship +from sqlalchemy.orm.collections import InstrumentedList Base = declarative_base() @@ -20,6 +21,31 @@ def to_dict(self): dictionary[column.name] = str(getattr(self, column.name)) return dictionary + def to_nested_dict(self, included_relations): + dictionary = {} + for column in self.__table__.columns: + dictionary[column.name] = str(getattr(self, column.name)) + if type(included_relations) is not dict: + for attr in dir(self): + if attr in included_relations: + relation = getattr(self, attr) + if isinstance(relation, EntityHelper): + dictionary[attr + "_ID"] = relation.to_dict() # if this was .to_nested_dict() it will make dictionaries all the way down + elif isinstance(relation, InstrumentedList): # Instrumented list is when the inclusion is a child + dictionary[attr + "_ID"] = [] + for entity in getattr(self, attr): + dictionary[attr + "_ID"].append(entity.to_dict()) + else: + for attr in dir(self): + print(included_relations.keys()) + print(included_relations.values()) + if attr == list(included_relations.keys())[0]: + dictionary[attr + "_ID"] = getattr(self, attr).to_nested_dict(list(included_relations.values())) + + dictionary = {k: v for k, v in dictionary.items() if + "ID" in k or k != "MOD_ID" or k != "CREATE_ID" or k != "ID"} + return dictionary + def update_from_dict(self, dictionary): """ Given a dictionary containing field names and variables, updates the entity from the given dictionary From fb0fb90e4621a6b1c09df587c1d21a673a673bbe Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 08:03:24 +0100 Subject: [PATCH 15/18] #2: Use nested dict method --- common/database_helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 9bb6b6f0..7accb6aa 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -172,9 +172,12 @@ def get_rows_by_filter(table, filters): results = base_query.all() # check if include was provided, then add included results if includes_relation: + log.info(" Closing DB session") for query_filter in filters: if list(query_filter)[0] == "include": - results = get_related_entities(query_filter["include"], results) + return list(map(lambda x: x.to_nested_dict(query_filter["include"]), results)) + + log.info(" Closing DB session") session.close() return list(map(lambda x: x.to_dict(), results)) From d9716afcd22ac99adee6f6bbbc95b3af4cc62d0f Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 08:05:17 +0100 Subject: [PATCH 16/18] #2: Remove print statement --- common/models/db_models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/models/db_models.py b/common/models/db_models.py index 6b92bf89..68619bbc 100644 --- a/common/models/db_models.py +++ b/common/models/db_models.py @@ -37,8 +37,6 @@ def to_nested_dict(self, included_relations): dictionary[attr + "_ID"].append(entity.to_dict()) else: for attr in dir(self): - print(included_relations.keys()) - print(included_relations.values()) if attr == list(included_relations.keys())[0]: dictionary[attr + "_ID"] = getattr(self, attr).to_nested_dict(list(included_relations.values())) From a9c83d7c2ff1f585ae368aaf879aaf21d510f2ea Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 08:40:06 +0100 Subject: [PATCH 17/18] #2: Add docstring to method --- common/models/db_models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/common/models/db_models.py b/common/models/db_models.py index 68619bbc..ea49eddf 100644 --- a/common/models/db_models.py +++ b/common/models/db_models.py @@ -22,6 +22,13 @@ def to_dict(self): return dictionary def to_nested_dict(self, included_relations): + """ + Given related models return a nested dictionary with the child or parent rows nested. + + + :param included_relations: string/list/dict - The related models to include. + :return: A nested dictionary with the included models + """ dictionary = {} for column in self.__table__.columns: dictionary[column.name] = str(getattr(self, column.name)) @@ -30,7 +37,7 @@ def to_nested_dict(self, included_relations): if attr in included_relations: relation = getattr(self, attr) if isinstance(relation, EntityHelper): - dictionary[attr + "_ID"] = relation.to_dict() # if this was .to_nested_dict() it will make dictionaries all the way down + dictionary[attr + "_ID"] = relation.to_dict() elif isinstance(relation, InstrumentedList): # Instrumented list is when the inclusion is a child dictionary[attr + "_ID"] = [] for entity in getattr(self, attr): From ed6ebd802df7f93ef1696fc5133d86c9f6ea0ece Mon Sep 17 00:00:00 2001 From: Keiran Price Date: Mon, 22 Jul 2019 14:44:29 +0100 Subject: [PATCH 18/18] #2: Remove repeated arg --- common/database_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/database_helpers.py b/common/database_helpers.py index 0268ab4f..8fd22ce3 100644 --- a/common/database_helpers.py +++ b/common/database_helpers.py @@ -133,7 +133,7 @@ def get_rows_by_filter(table, filters): where_part = query_filter[key] for k in where_part: column = getattr(table, k.upper()) - base_query = base_query.filter(column.in_([where_part[k]]), column.in_([where_part[k]])) + base_query = base_query.filter(column.in_([where_part[k]])) elif list(query_filter)[0].lower() == "order": for key in query_filter: field = query_filter[key].split(" ")[0]