From 7401b5e31a0a7ede619f1d552de045cd9e448194 Mon Sep 17 00:00:00 2001 From: Patrick Kalita Date: Tue, 23 Jul 2024 10:57:32 -0700 Subject: [PATCH 1/5] Upgrade to latest schema versions --- pyproject.toml | 4 ++-- web/package.json | 2 +- web/yarn.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fa730981..b9a0d604 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,8 @@ dependencies = [ "ipython==7.31.1", "itsdangerous==2.0.1", "mypy<0.920", - "nmdc-schema==10.5.5", - "nmdc-submission-schema==10.5.1", + "nmdc-schema==10.6.0", + "nmdc-submission-schema==10.6.0", "pint==0.18", "psycopg2==2.9.3", "pydantic==1.10.2", diff --git a/web/package.json b/web/package.json index 68385788..becf6463 100644 --- a/web/package.json +++ b/web/package.json @@ -34,7 +34,7 @@ "linkify-it": "^4.0.1", "lodash": "^4.17.21", "moment": "^2.29.4", - "nmdc-schema": "https://github.com/microbiomedata/nmdc-schema#v10.5.5", + "nmdc-schema": "https://github.com/microbiomedata/nmdc-schema#v10.6.0", "popper.js": "1.16.1", "protobufjs": "^6.11.3", "serialize-javascript": "^6.0.0", diff --git a/web/yarn.lock b/web/yarn.lock index 584fe97d..9cfaf684 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -7635,9 +7635,9 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -"nmdc-schema@https://github.com/microbiomedata/nmdc-schema#v10.5.5": +"nmdc-schema@https://github.com/microbiomedata/nmdc-schema#v10.6.0": version "0.0.0" - resolved "https://github.com/microbiomedata/nmdc-schema#0ed6dfd1823deaa8ede6ca8250188429d7d86305" + resolved "https://github.com/microbiomedata/nmdc-schema#3e430381daf40708d523781a6e0e7a84c70df4b5" no-case@^2.2.0: version "2.3.2" From 5f0ceb72f5cdce6fbd38487dda7d4f0d04c4b1d5 Mon Sep 17 00:00:00 2001 From: Patrick Kalita Date: Tue, 23 Jul 2024 12:03:10 -0700 Subject: [PATCH 2/5] Do not static JSON files for indivudual schema classes --- nmdc_server/static_files.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nmdc_server/static_files.py b/nmdc_server/static_files.py index 2dde1eb3..5509c7dc 100644 --- a/nmdc_server/static_files.py +++ b/nmdc_server/static_files.py @@ -38,10 +38,6 @@ def generate_submission_schema_files(directory: Path) -> None: # The entire submission schema in JSON format json_dumper.dump(sv.schema, out_dir / "submission_schema.json") - # Each class of the submission schema in JSON format - for class_name, class_definition in sv.all_classes().items(): - json_dumper.dump(class_definition, out_dir / f"{class_name}.json") - # The GOLD ecosystem tree that the submission schema re-exports gold_tree_path = submission_schema_files / "project/thirdparty/GoldEcosystemTree.json" shutil.copyfile(str(gold_tree_path), out_dir / "GoldEcosystemTree.json") From 3ec5dd63c0ba9b59225ec19a0279514bfd3448e7 Mon Sep 17 00:00:00 2001 From: Patrick Kalita Date: Tue, 23 Jul 2024 12:04:00 -0700 Subject: [PATCH 3/5] Use schema annotations to determine worksheet names when importing/exporting Excel files --- .../views/SubmissionPortal/HarmonizerView.vue | 24 +++++++++++++++++-- .../views/SubmissionPortal/harmonizerApi.ts | 17 +++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/web/src/views/SubmissionPortal/HarmonizerView.vue b/web/src/views/SubmissionPortal/HarmonizerView.vue index e8ba2c1f..e5571089 100644 --- a/web/src/views/SubmissionPortal/HarmonizerView.vue +++ b/web/src/views/SubmissionPortal/HarmonizerView.vue @@ -123,6 +123,8 @@ export default defineComponent({ const submitDialog = ref(false); const snackbar = ref(false); + const importErrorSnackbar = ref(false); + const notImportedWorksheetNames = ref([] as string[]); watch(activeTemplate, () => { // WARNING: It's important to do the column settings update /before/ data. Otherwise, @@ -384,7 +386,7 @@ export default defineComponent({ ], { skipHeader: true, }); - utils.book_append_sheet(workbook, worksheet, template.displayName); + utils.book_append_sheet(workbook, worksheet, template.excelWorksheetName || template.displayName); }); writeFile(workbook, EXPORT_FILENAME, { compression: true }); } @@ -401,9 +403,13 @@ export default defineComponent({ } const workbook = read(event.target.result); const imported = {} as Record; + const notImported = [] as string[]; Object.entries(workbook.Sheets).forEach(([name, worksheet]) => { - const template = Object.values(HARMONIZER_TEMPLATES).find((template) => template.displayName === name); + const template = Object.values(HARMONIZER_TEMPLATES).find((template) => ( + template.excelWorksheetName === name || template.displayName === name + )); if (!template || !template.sampleDataSlot || !template.schemaClass) { + notImported.push(name); return; } @@ -422,6 +428,11 @@ export default defineComponent({ template.schemaClass, ); }); + + // Alert the user if any worksheets were not imported + notImportedWorksheetNames.value = notImported; + importErrorSnackbar.value = notImported.length > 0; + // Load imported data sampleData.value = imported; @@ -497,6 +508,8 @@ export default defineComponent({ submitDialog, snackbar, schemaLoading, + importErrorSnackbar, + notImportedWorksheetNames, /* methods */ doSubmit, downloadSamples, @@ -572,6 +585,13 @@ export default defineComponent({ > Validation Passed! You can now submit or continue editing. + + The following worksheet names were not recognized: {{ notImportedWorksheetNames.join(', ') }} + = { air: { @@ -208,6 +210,21 @@ export class HarmonizerApi { this.schema = schema; this.goldEcosystemTree = goldEcosystemTree; + // Attempt to find each template's underlying schema class, pull the excel_worksheet_name annotation from it + // and add it to the template object. + Object.values(HARMONIZER_TEMPLATES).forEach((template) => { + if (!template.schemaClass) { + return; + } + const classDefinition = schema.classes[template.schemaClass]; + if (!classDefinition) { + console.warn(`Template ${template.displayName} references class ${template.schemaClass} which is not defined in the schema`); + return; + } + // eslint-disable-next-line no-param-reassign + template.excelWorksheetName = classDefinition.annotations?.excel_worksheet_name?.value; + }); + this.dh = new DataHarmonizer(r, { modalsRoot: document.querySelector('.harmonizer-style-container'), fieldSettings: this._getFieldSettings(), From 485e381172bafc75f04c7fc2c8f01213efc628ba Mon Sep 17 00:00:00 2001 From: Patrick Kalita Date: Tue, 23 Jul 2024 12:09:04 -0700 Subject: [PATCH 4/5] Remove console.warn call --- web/src/views/SubmissionPortal/harmonizerApi.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/src/views/SubmissionPortal/harmonizerApi.ts b/web/src/views/SubmissionPortal/harmonizerApi.ts index a1104a70..b3e38877 100644 --- a/web/src/views/SubmissionPortal/harmonizerApi.ts +++ b/web/src/views/SubmissionPortal/harmonizerApi.ts @@ -218,7 +218,6 @@ export class HarmonizerApi { } const classDefinition = schema.classes[template.schemaClass]; if (!classDefinition) { - console.warn(`Template ${template.displayName} references class ${template.schemaClass} which is not defined in the schema`); return; } // eslint-disable-next-line no-param-reassign From 7737a1d14cef8d30ce49eb1785a9ea418fe606a9 Mon Sep 17 00:00:00 2001 From: Patrick Kalita Date: Wed, 24 Jul 2024 15:24:13 -0700 Subject: [PATCH 5/5] Upgrade to latest schema versions --- pyproject.toml | 4 ++-- web/package.json | 2 +- web/yarn.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b9a0d604..6ccb42c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,8 @@ dependencies = [ "ipython==7.31.1", "itsdangerous==2.0.1", "mypy<0.920", - "nmdc-schema==10.6.0", - "nmdc-submission-schema==10.6.0", + "nmdc-schema==10.7.0", + "nmdc-submission-schema==10.7.0", "pint==0.18", "psycopg2==2.9.3", "pydantic==1.10.2", diff --git a/web/package.json b/web/package.json index becf6463..51e23c6a 100644 --- a/web/package.json +++ b/web/package.json @@ -34,7 +34,7 @@ "linkify-it": "^4.0.1", "lodash": "^4.17.21", "moment": "^2.29.4", - "nmdc-schema": "https://github.com/microbiomedata/nmdc-schema#v10.6.0", + "nmdc-schema": "https://github.com/microbiomedata/nmdc-schema#v10.7.0", "popper.js": "1.16.1", "protobufjs": "^6.11.3", "serialize-javascript": "^6.0.0", diff --git a/web/yarn.lock b/web/yarn.lock index 9cfaf684..d4522165 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -7635,9 +7635,9 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -"nmdc-schema@https://github.com/microbiomedata/nmdc-schema#v10.6.0": +"nmdc-schema@https://github.com/microbiomedata/nmdc-schema#v10.7.0": version "0.0.0" - resolved "https://github.com/microbiomedata/nmdc-schema#3e430381daf40708d523781a6e0e7a84c70df4b5" + resolved "https://github.com/microbiomedata/nmdc-schema#ad1440736c1e088705a8b09f88727c0e195abfb3" no-case@^2.2.0: version "2.3.2"