From 06cddc37aa9d56c5df39eed7d65311e6771a8661 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 26 Nov 2024 10:09:25 -0500
Subject: [PATCH 01/22] first step to manifest jsonschema

---
 deployment/conf/deployment.cfg                |  3 ++-
 deployment/conf/testing.cfg                   |  3 ++-
 .../schema/dts_manifest.json                  | 26 +++++++++++++++++++
 staging_service/app.py                        |  9 +++++++
 staging_service/utils.py                      |  2 +-
 5 files changed, 40 insertions(+), 3 deletions(-)
 create mode 100644 import_specifications/schema/dts_manifest.json

diff --git a/deployment/conf/deployment.cfg b/deployment/conf/deployment.cfg
index be7851a4..5ec9ac74 100644
--- a/deployment/conf/deployment.cfg
+++ b/deployment/conf/deployment.cfg
@@ -3,4 +3,5 @@ META_DIR = /kb/deployment/lib/src/data/metadata/
 DATA_DIR = /kb/deployment/lib/src/data/bulk/
 AUTH_URL = https://ci.kbase.us/services/auth/api/V2/token
 CONCIERGE_PATH = /kbaseconcierge
-FILE_EXTENSION_MAPPINGS = /kb/deployment/conf/supported_apps_w_extensions.json
\ No newline at end of file
+FILE_EXTENSION_MAPPINGS = /kb/deployment/conf/supported_apps_w_extensions.json
+DTS_MANIFEST_SCHEMA = /kb/deployment/import_specifications/schema/dts_manifest.json
diff --git a/deployment/conf/testing.cfg b/deployment/conf/testing.cfg
index 432e99b7..41f067bb 100644
--- a/deployment/conf/testing.cfg
+++ b/deployment/conf/testing.cfg
@@ -4,4 +4,5 @@ DATA_DIR = ./data/bulk/
 AUTH_URL = https://ci.kbase.us/services/auth/api/V2/token
 CONCIERGE_PATH = /kbaseconcierge
 FILE_EXTENSION_MAPPINGS = ./deployment/conf/supported_apps_w_extensions.json
-;FILE_EXTENSION_MAPPINGS_PYCHARM = ../deployment/conf/supported_apps_w_extensions.json
\ No newline at end of file
+;FILE_EXTENSION_MAPPINGS_PYCHARM = ../deployment/conf/supported_apps_w_extensions.json
+DTS_MANIFEST_SCHEMA = ./import_specifications/schema/dts_manifest.json
diff --git a/import_specifications/schema/dts_manifest.json b/import_specifications/schema/dts_manifest.json
new file mode 100644
index 00000000..fc70d7ef
--- /dev/null
+++ b/import_specifications/schema/dts_manifest.json
@@ -0,0 +1,26 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "type": "object",
+    "properties": {
+        "resources": {"type": "array"},
+        "instructions": {
+            "type": "object",
+            "properties": {
+                "protocol": {"type": "string"},
+                "objects": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "data_type": {"type": "string"},
+                            "parameters": {"type": "object"}
+                        },
+                        "required": ["data_type", "parameters"]
+                    }
+                }
+            },
+            "required": ["protocol", "objects"]
+        }
+    },
+    "required": ["resources", "instructions"]
+}
diff --git a/staging_service/app.py b/staging_service/app.py
index cd668dc7..a3719bb2 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -599,6 +599,7 @@ def inject_config_dependencies(config):
     META_DIR = config["staging_service"]["META_DIR"]
     CONCIERGE_PATH = config["staging_service"]["CONCIERGE_PATH"]
     FILE_EXTENSION_MAPPINGS = config["staging_service"]["FILE_EXTENSION_MAPPINGS"]
+    DTS_MANIFEST_SCHEMA_PATH = config["staging_service"]["DTS_MANIFEST_SCHEMA"]
 
     if DATA_DIR.startswith("."):
         DATA_DIR = os.path.normpath(os.path.join(os.getcwd(), DATA_DIR))
@@ -610,10 +611,15 @@ def inject_config_dependencies(config):
         FILE_EXTENSION_MAPPINGS = os.path.normpath(
             os.path.join(os.getcwd(), FILE_EXTENSION_MAPPINGS)
         )
+    if DTS_MANIFEST_SCHEMA_PATH.startswith("."):
+        DTS_MANIFEST_SCHEMA_PATH = os.path.normpath(
+            os.path.join(os.getcwd(), DTS_MANIFEST_SCHEMA_PATH)
+        )
 
     Path._DATA_DIR = DATA_DIR
     Path._META_DIR = META_DIR
     Path._CONCIERGE_PATH = CONCIERGE_PATH
+    Path._DTS_MANIFEST_SCHEMA_PATH = DTS_MANIFEST_SCHEMA_PATH
 
     if Path._DATA_DIR is None:
         raise Exception("Please provide DATA_DIR in the config file ")
@@ -624,6 +630,9 @@ def inject_config_dependencies(config):
     if Path._CONCIERGE_PATH is None:
         raise Exception("Please provide CONCIERGE_PATH in the config file ")
 
+    if Path._DTS_MANIFEST_SCHEMA_PATH is None:
+        raise Exception("Please provide DTS_MANIFEST_SCHEMA in the config file")
+
     if FILE_EXTENSION_MAPPINGS is None:
         raise Exception("Please provide FILE_EXTENSION_MAPPINGS in the config file ")
     with open(FILE_EXTENSION_MAPPINGS, "r", encoding="utf-8") as file_extension_mappings_file:
diff --git a/staging_service/utils.py b/staging_service/utils.py
index 835ccedc..379fb929 100644
--- a/staging_service/utils.py
+++ b/staging_service/utils.py
@@ -44,6 +44,7 @@ class Path(object):
     _DATA_DIR = None  # expects to be set by config
     _CONCIERGE_PATH = None  # expects to be set by config
     _FILE_EXTENSION_MAPPINGS = None  # expects to be set by config
+    _DTS_MANIFEST_SCHEMA_PATH = None  # expects to be set by config
 
     __slots__ = ["full_path", "metadata_path", "user_path", "name", "jgi_metadata"]
 
@@ -85,7 +86,6 @@ def from_full_path(full_path: str):
         jgi_metadata = os.path.join(os.path.dirname(full_path), "." + name + ".jgi")
         return Path(full_path, metadata_path, user_path, name, jgi_metadata)
 
-
 class AclManager:
     def __init__(self):
         """

From 0893cfb3dbec97a466c170613e28f385b5da81f3 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 26 Nov 2024 10:09:56 -0500
Subject: [PATCH 02/22] change app and parser to support schema

---
 staging_service/app.py                                      | 5 ++++-
 staging_service/import_specifications/individual_parsers.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index a3719bb2..d9e983f5 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -41,6 +41,7 @@
 VERSION = "1.3.6"
 
 _DATATYPE_MAPPINGS = None
+_DTS_MANIFEST_SCHEMA = None
 
 _APP_JSON = "application/json"
 
@@ -48,7 +49,7 @@
     CSV: parse_csv,
     TSV: parse_tsv,
     EXCEL: parse_excel,
-    JSON: parse_dts_manifest,
+    JSON: lambda path: parse_dts_manifest(path, _DTS_MANIFEST_SCHEMA),
 }
 
 _IMPSPEC_FILE_TO_WRITER = {
@@ -632,6 +633,8 @@ def inject_config_dependencies(config):
 
     if Path._DTS_MANIFEST_SCHEMA_PATH is None:
         raise Exception("Please provide DTS_MANIFEST_SCHEMA in the config file")
+    global _DTS_MANIFEST_SCHEMA
+    _DTS_MANIFEST_SCHEMA = DTS_MANIFEST_SCHEMA_PATH
 
     if FILE_EXTENSION_MAPPINGS is None:
         raise Exception("Please provide FILE_EXTENSION_MAPPINGS in the config file ")
diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index bf98a52b..73e78c46 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -319,7 +319,7 @@ def parse_excel(path: Path) -> ParseResults:
         return _error(Error(ErrorType.PARSE_FAIL, "No non-header data in file", spcsrc))
 
 
-def parse_dts_manifest(path: Path) -> ParseResults:
+def parse_dts_manifest(path: Path, dts_manifest_schema: Path) -> ParseResults:
     """
     Parse the provided DTS manifest file. Expected to be JSON, and will fail otherwise.
     The manifest should have this format, with expected keys included:

From b2f87bd7ec6c8f53a10ca918927d266a006b33ee Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 3 Dec 2024 13:44:06 -0500
Subject: [PATCH 03/22] convert to jsonschema

---
 Pipfile                                       |   1 +
 Pipfile.lock                                  | 676 ++++++++++--------
 requirements.txt                              |   1 +
 staging_service/app.py                        |  18 +-
 .../individual_parsers.py                     |  15 +-
 .../test_individual_parsers.py                |  46 +-
 6 files changed, 448 insertions(+), 309 deletions(-)

diff --git a/Pipfile b/Pipfile
index c5327fc4..5d71a0d7 100644
--- a/Pipfile
+++ b/Pipfile
@@ -16,6 +16,7 @@ defusedxml = "==0.7.1"
 frozendict = "==2.3.8"
 globus-sdk = "==1.6.1"
 hypothesis = "==6.81.1"
+jsonschema = "==4.23.0"
 openpyxl = "==3.1.2"
 pandas = "==2.2.3"
 pytest = "==7.4.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index abd8d830..4e00fb18 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "37fa5e2c5c7c5c35e27e6dd6d73c8f089084bc799ff50cea9c9ecefef9a6478c"
+            "sha256": "7f1b645c0bb01cbfcc2f0780d528a31aec8eb360823d48b591b9e13adce7db93"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -451,35 +451,37 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362",
-                "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4",
-                "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa",
-                "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83",
-                "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff",
-                "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805",
-                "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6",
-                "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664",
-                "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08",
-                "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e",
-                "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18",
-                "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f",
-                "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73",
-                "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5",
-                "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984",
-                "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd",
-                "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3",
-                "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e",
-                "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405",
-                "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2",
-                "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c",
-                "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995",
-                "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73",
-                "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16",
-                "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7",
-                "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd",
-                "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7"
+                "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7",
+                "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731",
+                "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b",
+                "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc",
+                "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543",
+                "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385",
+                "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c",
+                "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591",
+                "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede",
+                "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb",
+                "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f",
+                "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123",
+                "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c",
+                "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba",
+                "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c",
+                "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285",
+                "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd",
+                "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092",
+                "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa",
+                "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289",
+                "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02",
+                "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64",
+                "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053",
+                "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417",
+                "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e",
+                "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e",
+                "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7",
+                "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756",
+                "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4"
             ],
-            "version": "==43.0.3"
+            "version": "==44.0.0"
         },
         "defusedxml": {
             "hashes": [
@@ -498,14 +500,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.0.0"
         },
-        "exceptiongroup": {
-            "hashes": [
-                "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
-                "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
-            ],
-            "markers": "python_version < '3.11'",
-            "version": "==1.2.2"
-        },
         "frozendict": {
             "hashes": [
                 "sha256:0bc4767e2f83db5b701c787e22380296977368b0c57e485ca71b2eedfa11c4a3",
@@ -681,6 +675,23 @@
             "markers": "python_version >= '3.7'",
             "version": "==2.0.0"
         },
+        "jsonschema": {
+            "hashes": [
+                "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4",
+                "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==4.23.0"
+        },
+        "jsonschema-specifications": {
+            "hashes": [
+                "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272",
+                "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2024.10.1"
+        },
         "multidict": {
             "hashes": [
                 "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f",
@@ -789,54 +800,64 @@
         },
         "numpy": {
             "hashes": [
-                "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a",
-                "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195",
-                "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951",
-                "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1",
-                "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c",
-                "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc",
-                "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b",
-                "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd",
-                "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4",
-                "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd",
-                "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318",
-                "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448",
-                "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece",
-                "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d",
-                "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5",
-                "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8",
-                "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57",
-                "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78",
-                "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66",
-                "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a",
-                "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e",
-                "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c",
-                "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa",
-                "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d",
-                "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c",
-                "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729",
-                "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97",
-                "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c",
-                "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9",
-                "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669",
-                "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4",
-                "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73",
-                "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385",
-                "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8",
-                "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c",
-                "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b",
-                "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692",
-                "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15",
-                "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131",
-                "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a",
-                "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326",
-                "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b",
-                "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded",
-                "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04",
-                "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"
+                "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe",
+                "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0",
+                "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48",
+                "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a",
+                "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564",
+                "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958",
+                "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17",
+                "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0",
+                "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee",
+                "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b",
+                "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4",
+                "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4",
+                "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6",
+                "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4",
+                "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d",
+                "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f",
+                "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f",
+                "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f",
+                "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56",
+                "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9",
+                "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd",
+                "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23",
+                "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed",
+                "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a",
+                "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098",
+                "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1",
+                "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512",
+                "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f",
+                "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09",
+                "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f",
+                "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc",
+                "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8",
+                "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0",
+                "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761",
+                "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef",
+                "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5",
+                "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e",
+                "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b",
+                "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d",
+                "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43",
+                "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c",
+                "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41",
+                "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff",
+                "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408",
+                "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2",
+                "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9",
+                "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57",
+                "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb",
+                "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9",
+                "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3",
+                "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a",
+                "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0",
+                "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e",
+                "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598",
+                "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"
             ],
-            "markers": "python_version < '3.11'",
-            "version": "==2.0.2"
+            "markers": "python_version == '3.11'",
+            "version": "==2.1.3"
         },
         "openpyxl": {
             "hashes": [
@@ -849,11 +870,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
-                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
+                "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759",
+                "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==24.1"
+            "version": "==24.2"
         },
         "pandas": {
             "hashes": [
@@ -930,107 +951,91 @@
         },
         "propcache": {
             "hashes": [
-                "sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9",
-                "sha256:0e53cb83fdd61cbd67202735e6a6687a7b491c8742dfc39c9e01e80354956763",
-                "sha256:1235c01ddaa80da8235741e80815ce381c5267f96cc49b1477fdcf8c047ef325",
-                "sha256:140fbf08ab3588b3468932974a9331aff43c0ab8a2ec2c608b6d7d1756dbb6cb",
-                "sha256:191db28dc6dcd29d1a3e063c3be0b40688ed76434622c53a284e5427565bbd9b",
-                "sha256:1e41d67757ff4fbc8ef2af99b338bfb955010444b92929e9e55a6d4dcc3c4f09",
-                "sha256:1ec43d76b9677637a89d6ab86e1fef70d739217fefa208c65352ecf0282be957",
-                "sha256:20a617c776f520c3875cf4511e0d1db847a076d720714ae35ffe0df3e440be68",
-                "sha256:218db2a3c297a3768c11a34812e63b3ac1c3234c3a086def9c0fee50d35add1f",
-                "sha256:22aa8f2272d81d9317ff5756bb108021a056805ce63dd3630e27d042c8092798",
-                "sha256:25a1f88b471b3bc911d18b935ecb7115dff3a192b6fef46f0bfaf71ff4f12418",
-                "sha256:25c8d773a62ce0451b020c7b29a35cfbc05de8b291163a7a0f3b7904f27253e6",
-                "sha256:2a60ad3e2553a74168d275a0ef35e8c0a965448ffbc3b300ab3a5bb9956c2162",
-                "sha256:2a66df3d4992bc1d725b9aa803e8c5a66c010c65c741ad901e260ece77f58d2f",
-                "sha256:2ccc28197af5313706511fab3a8b66dcd6da067a1331372c82ea1cb74285e036",
-                "sha256:2e900bad2a8456d00a113cad8c13343f3b1f327534e3589acc2219729237a2e8",
-                "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2",
-                "sha256:33ac8f098df0585c0b53009f039dfd913b38c1d2edafed0cedcc0c32a05aa110",
-                "sha256:3444cdba6628accf384e349014084b1cacd866fbb88433cd9d279d90a54e0b23",
-                "sha256:363ea8cd3c5cb6679f1c2f5f1f9669587361c062e4899fce56758efa928728f8",
-                "sha256:375a12d7556d462dc64d70475a9ee5982465fbb3d2b364f16b86ba9135793638",
-                "sha256:388f3217649d6d59292b722d940d4d2e1e6a7003259eb835724092a1cca0203a",
-                "sha256:3947483a381259c06921612550867b37d22e1df6d6d7e8361264b6d037595f44",
-                "sha256:39e104da444a34830751715f45ef9fc537475ba21b7f1f5b0f4d71a3b60d7fe2",
-                "sha256:3c997f8c44ec9b9b0bcbf2d422cc00a1d9b9c681f56efa6ca149a941e5560da2",
-                "sha256:3dfafb44f7bb35c0c06eda6b2ab4bfd58f02729e7c4045e179f9a861b07c9850",
-                "sha256:3ebbcf2a07621f29638799828b8d8668c421bfb94c6cb04269130d8de4fb7136",
-                "sha256:3f88a4095e913f98988f5b338c1d4d5d07dbb0b6bad19892fd447484e483ba6b",
-                "sha256:439e76255daa0f8151d3cb325f6dd4a3e93043e6403e6491813bcaaaa8733887",
-                "sha256:4569158070180c3855e9c0791c56be3ceeb192defa2cdf6a3f39e54319e56b89",
-                "sha256:466c219deee4536fbc83c08d09115249db301550625c7fef1c5563a584c9bc87",
-                "sha256:4a9d9b4d0a9b38d1c391bb4ad24aa65f306c6f01b512e10a8a34a2dc5675d348",
-                "sha256:4c7dde9e533c0a49d802b4f3f218fa9ad0a1ce21f2c2eb80d5216565202acab4",
-                "sha256:53d1bd3f979ed529f0805dd35ddaca330f80a9a6d90bc0121d2ff398f8ed8861",
-                "sha256:55346705687dbd7ef0d77883ab4f6fabc48232f587925bdaf95219bae072491e",
-                "sha256:56295eb1e5f3aecd516d91b00cfd8bf3a13991de5a479df9e27dd569ea23959c",
-                "sha256:56bb5c98f058a41bb58eead194b4db8c05b088c93d94d5161728515bd52b052b",
-                "sha256:5a5b3bb545ead161be780ee85a2b54fdf7092815995661947812dde94a40f6fb",
-                "sha256:5f2564ec89058ee7c7989a7b719115bdfe2a2fb8e7a4543b8d1c0cc4cf6478c1",
-                "sha256:608cce1da6f2672a56b24a015b42db4ac612ee709f3d29f27a00c943d9e851de",
-                "sha256:63f13bf09cc3336eb04a837490b8f332e0db41da66995c9fd1ba04552e516354",
-                "sha256:662dd62358bdeaca0aee5761de8727cfd6861432e3bb828dc2a693aa0471a563",
-                "sha256:676135dcf3262c9c5081cc8f19ad55c8a64e3f7282a21266d05544450bffc3a5",
-                "sha256:67aeb72e0f482709991aa91345a831d0b707d16b0257e8ef88a2ad246a7280bf",
-                "sha256:67b69535c870670c9f9b14a75d28baa32221d06f6b6fa6f77a0a13c5a7b0a5b9",
-                "sha256:682a7c79a2fbf40f5dbb1eb6bfe2cd865376deeac65acf9beb607505dced9e12",
-                "sha256:6994984550eaf25dd7fc7bd1b700ff45c894149341725bb4edc67f0ffa94efa4",
-                "sha256:69d3a98eebae99a420d4b28756c8ce6ea5a29291baf2dc9ff9414b42676f61d5",
-                "sha256:6e2e54267980349b723cff366d1e29b138b9a60fa376664a157a342689553f71",
-                "sha256:73e4b40ea0eda421b115248d7e79b59214411109a5bc47d0d48e4c73e3b8fcf9",
-                "sha256:74acd6e291f885678631b7ebc85d2d4aec458dd849b8c841b57ef04047833bed",
-                "sha256:7665f04d0c7f26ff8bb534e1c65068409bf4687aa2534faf7104d7182debb336",
-                "sha256:7735e82e3498c27bcb2d17cb65d62c14f1100b71723b68362872bca7d0913d90",
-                "sha256:77a86c261679ea5f3896ec060be9dc8e365788248cc1e049632a1be682442063",
-                "sha256:7cf18abf9764746b9c8704774d8b06714bcb0a63641518a3a89c7f85cc02c2ad",
-                "sha256:83928404adf8fb3d26793665633ea79b7361efa0287dfbd372a7e74311d51ee6",
-                "sha256:8e40876731f99b6f3c897b66b803c9e1c07a989b366c6b5b475fafd1f7ba3fb8",
-                "sha256:8f188cfcc64fb1266f4684206c9de0e80f54622c3f22a910cbd200478aeae61e",
-                "sha256:91997d9cb4a325b60d4e3f20967f8eb08dfcb32b22554d5ef78e6fd1dda743a2",
-                "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7",
-                "sha256:92fe151145a990c22cbccf9ae15cae8ae9eddabfc949a219c9f667877e40853d",
-                "sha256:945db8ee295d3af9dbdbb698cce9bbc5c59b5c3fe328bbc4387f59a8a35f998d",
-                "sha256:9517d5e9e0731957468c29dbfd0f976736a0e55afaea843726e887f36fe017df",
-                "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b",
-                "sha256:97a58a28bcf63284e8b4d7b460cbee1edaab24634e82059c7b8c09e65284f178",
-                "sha256:97e48e8875e6c13909c800fa344cd54cc4b2b0db1d5f911f840458a500fde2c2",
-                "sha256:9e0f07b42d2a50c7dd2d8675d50f7343d998c64008f1da5fef888396b7f84630",
-                "sha256:a3dc1a4b165283bd865e8f8cb5f0c64c05001e0718ed06250d8cac9bec115b48",
-                "sha256:a3ebe9a75be7ab0b7da2464a77bb27febcb4fab46a34f9288f39d74833db7f61",
-                "sha256:a64e32f8bd94c105cc27f42d3b658902b5bcc947ece3c8fe7bc1b05982f60e89",
-                "sha256:a6ed8db0a556343d566a5c124ee483ae113acc9a557a807d439bcecc44e7dfbb",
-                "sha256:ad9c9b99b05f163109466638bd30ada1722abb01bbb85c739c50b6dc11f92dc3",
-                "sha256:b33d7a286c0dc1a15f5fc864cc48ae92a846df287ceac2dd499926c3801054a6",
-                "sha256:bc092ba439d91df90aea38168e11f75c655880c12782facf5cf9c00f3d42b562",
-                "sha256:c436130cc779806bdf5d5fae0d848713105472b8566b75ff70048c47d3961c5b",
-                "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58",
-                "sha256:c5ecca8f9bab618340c8e848d340baf68bcd8ad90a8ecd7a4524a81c1764b3db",
-                "sha256:cfac69017ef97db2438efb854edf24f5a29fd09a536ff3a992b75990720cdc99",
-                "sha256:d2f0d0f976985f85dfb5f3d685697ef769faa6b71993b46b295cdbbd6be8cc37",
-                "sha256:d5bed7f9805cc29c780f3aee05de3262ee7ce1f47083cfe9f77471e9d6777e83",
-                "sha256:d6a21ef516d36909931a2967621eecb256018aeb11fc48656e3257e73e2e247a",
-                "sha256:d9b6ddac6408194e934002a69bcaadbc88c10b5f38fb9307779d1c629181815d",
-                "sha256:db47514ffdbd91ccdc7e6f8407aac4ee94cc871b15b577c1c324236b013ddd04",
-                "sha256:df81779732feb9d01e5d513fad0122efb3d53bbc75f61b2a4f29a020bc985e70",
-                "sha256:e4a91d44379f45f5e540971d41e4626dacd7f01004826a18cb048e7da7e96544",
-                "sha256:e63e3e1e0271f374ed489ff5ee73d4b6e7c60710e1f76af5f0e1a6117cd26394",
-                "sha256:e70fac33e8b4ac63dfc4c956fd7d85a0b1139adcfc0d964ce288b7c527537fea",
-                "sha256:ecddc221a077a8132cf7c747d5352a15ed763b674c0448d811f408bf803d9ad7",
-                "sha256:f45eec587dafd4b2d41ac189c2156461ebd0c1082d2fe7013571598abb8505d1",
-                "sha256:f52a68c21363c45297aca15561812d542f8fc683c85201df0bebe209e349f793",
-                "sha256:f571aea50ba5623c308aa146eb650eebf7dbe0fd8c5d946e28343cb3b5aad577",
-                "sha256:f60f0ac7005b9f5a6091009b09a419ace1610e163fa5deaba5ce3484341840e7",
-                "sha256:f6475a1b2ecb310c98c28d271a30df74f9dd436ee46d09236a6b750a7599ce57",
-                "sha256:f6d5749fdd33d90e34c2efb174c7e236829147a2713334d708746e94c4bde40d",
-                "sha256:f902804113e032e2cdf8c71015651c97af6418363bea8d78dc0911d56c335032",
-                "sha256:fa1076244f54bb76e65e22cb6910365779d5c3d71d1f18b275f1dfc7b0d71b4d",
-                "sha256:fc2db02409338bf36590aa985a461b2c96fce91f8e7e0f14c50c5fcc4f229016",
-                "sha256:ffcad6c564fe6b9b8916c1aefbb37a362deebf9394bd2974e9d84232e3e08504"
+                "sha256:03ff9d3f665769b2a85e6157ac8b439644f2d7fd17615a82fa55739bc97863f4",
+                "sha256:049324ee97bb67285b49632132db351b41e77833678432be52bdd0289c0e05e4",
+                "sha256:081a430aa8d5e8876c6909b67bd2d937bfd531b0382d3fdedb82612c618bc41a",
+                "sha256:0f022d381747f0dfe27e99d928e31bc51a18b65bb9e481ae0af1380a6725dd1f",
+                "sha256:12d1083f001ace206fe34b6bdc2cb94be66d57a850866f0b908972f90996b3e9",
+                "sha256:14d86fe14b7e04fa306e0c43cdbeebe6b2c2156a0c9ce56b815faacc193e320d",
+                "sha256:160291c60081f23ee43d44b08a7e5fb76681221a8e10b3139618c5a9a291b84e",
+                "sha256:1672137af7c46662a1c2be1e8dc78cb6d224319aaa40271c9257d886be4363a6",
+                "sha256:19a0f89a7bb9d8048d9c4370c9c543c396e894c76be5525f5e1ad287f1750ddf",
+                "sha256:1ac2f5fe02fa75f56e1ad473f1175e11f475606ec9bd0be2e78e4734ad575034",
+                "sha256:1cd9a1d071158de1cc1c71a26014dcdfa7dd3d5f4f88c298c7f90ad6f27bb46d",
+                "sha256:1ffc3cca89bb438fb9c95c13fc874012f7b9466b89328c3c8b1aa93cdcfadd16",
+                "sha256:297878dc9d0a334358f9b608b56d02e72899f3b8499fc6044133f0d319e2ec30",
+                "sha256:2d3af2e79991102678f53e0dbf4c35de99b6b8b58f29a27ca0325816364caaba",
+                "sha256:30b43e74f1359353341a7adb783c8f1b1c676367b011709f466f42fda2045e95",
+                "sha256:3156628250f46a0895f1f36e1d4fbe062a1af8718ec3ebeb746f1d23f0c5dc4d",
+                "sha256:31f5af773530fd3c658b32b6bdc2d0838543de70eb9a2156c03e410f7b0d3aae",
+                "sha256:3935bfa5fede35fb202c4b569bb9c042f337ca4ff7bd540a0aa5e37131659348",
+                "sha256:39d51fbe4285d5db5d92a929e3e21536ea3dd43732c5b177c7ef03f918dff9f2",
+                "sha256:3f77ce728b19cb537714499928fe800c3dda29e8d9428778fc7c186da4c09a64",
+                "sha256:4160d9283bd382fa6c0c2b5e017acc95bc183570cd70968b9202ad6d8fc48dce",
+                "sha256:4a571d97dbe66ef38e472703067021b1467025ec85707d57e78711c085984e54",
+                "sha256:4e6281aedfca15301c41f74d7005e6e3f4ca143584ba696ac69df4f02f40d629",
+                "sha256:52277518d6aae65536e9cea52d4e7fd2f7a66f4aa2d30ed3f2fcea620ace3c54",
+                "sha256:556fc6c10989f19a179e4321e5d678db8eb2924131e64652a51fe83e4c3db0e1",
+                "sha256:574faa3b79e8ebac7cb1d7930f51184ba1ccf69adfdec53a12f319a06030a68b",
+                "sha256:58791550b27d5488b1bb52bc96328456095d96206a250d28d874fafe11b3dfaf",
+                "sha256:5b750a8e5a1262434fb1517ddf64b5de58327f1adc3524a5e44c2ca43305eb0b",
+                "sha256:5d97151bc92d2b2578ff7ce779cdb9174337390a535953cbb9452fb65164c587",
+                "sha256:5eee736daafa7af6d0a2dc15cc75e05c64f37fc37bafef2e00d77c14171c2097",
+                "sha256:6445804cf4ec763dc70de65a3b0d9954e868609e83850a47ca4f0cb64bd79fea",
+                "sha256:647894f5ae99c4cf6bb82a1bb3a796f6e06af3caa3d32e26d2350d0e3e3faf24",
+                "sha256:66d4cfda1d8ed687daa4bc0274fcfd5267873db9a5bc0418c2da19273040eeb7",
+                "sha256:6a9a8c34fb7bb609419a211e59da8887eeca40d300b5ea8e56af98f6fbbb1541",
+                "sha256:6b3f39a85d671436ee3d12c017f8fdea38509e4f25b28eb25877293c98c243f6",
+                "sha256:6b6fb63ae352e13748289f04f37868099e69dba4c2b3e271c46061e82c745634",
+                "sha256:70693319e0b8fd35dd863e3e29513875eb15c51945bf32519ef52927ca883bc3",
+                "sha256:781e65134efaf88feb447e8c97a51772aa75e48b794352f94cb7ea717dedda0d",
+                "sha256:819ce3b883b7576ca28da3861c7e1a88afd08cc8c96908e08a3f4dd64a228034",
+                "sha256:857112b22acd417c40fa4595db2fe28ab900c8c5fe4670c7989b1c0230955465",
+                "sha256:887d9b0a65404929641a9fabb6452b07fe4572b269d901d622d8a34a4e9043b2",
+                "sha256:8b3489ff1ed1e8315674d0775dc7d2195fb13ca17b3808721b54dbe9fd020faf",
+                "sha256:92fc4500fcb33899b05ba73276dfb684a20d31caa567b7cb5252d48f896a91b1",
+                "sha256:9403db39be1393618dd80c746cb22ccda168efce239c73af13c3763ef56ffc04",
+                "sha256:98110aa363f1bb4c073e8dcfaefd3a5cea0f0834c2aab23dda657e4dab2f53b5",
+                "sha256:999779addc413181912e984b942fbcc951be1f5b3663cd80b2687758f434c583",
+                "sha256:9caac6b54914bdf41bcc91e7eb9147d331d29235a7c967c150ef5df6464fd1bb",
+                "sha256:a7a078f5d37bee6690959c813977da5291b24286e7b962e62a94cec31aa5188b",
+                "sha256:a7e65eb5c003a303b94aa2c3852ef130230ec79e349632d030e9571b87c4698c",
+                "sha256:a96dc1fa45bd8c407a0af03b2d5218392729e1822b0c32e62c5bf7eeb5fb3958",
+                "sha256:aca405706e0b0a44cc6bfd41fbe89919a6a56999157f6de7e182a990c36e37bc",
+                "sha256:accb6150ce61c9c4b7738d45550806aa2b71c7668c6942f17b0ac182b6142fd4",
+                "sha256:ad1af54a62ffe39cf34db1aa6ed1a1873bd548f6401db39d8e7cd060b9211f82",
+                "sha256:ae1aa1cd222c6d205853b3013c69cd04515f9d6ab6de4b0603e2e1c33221303e",
+                "sha256:b2d0a12018b04f4cb820781ec0dffb5f7c7c1d2a5cd22bff7fb055a2cb19ebce",
+                "sha256:b480c6a4e1138e1aa137c0079b9b6305ec6dcc1098a8ca5196283e8a49df95a9",
+                "sha256:b74c261802d3d2b85c9df2dfb2fa81b6f90deeef63c2db9f0e029a3cac50b518",
+                "sha256:ba278acf14471d36316159c94a802933d10b6a1e117b8554fe0d0d9b75c9d536",
+                "sha256:bb6178c241278d5fe853b3de743087be7f5f4c6f7d6d22a3b524d323eecec505",
+                "sha256:bf72af5e0fb40e9babf594308911436c8efde3cb5e75b6f206c34ad18be5c052",
+                "sha256:bfd3223c15bebe26518d58ccf9a39b93948d3dcb3e57a20480dfdd315356baff",
+                "sha256:c214999039d4f2a5b2073ac506bba279945233da8c786e490d411dfc30f855c1",
+                "sha256:c2f992c07c0fca81655066705beae35fc95a2fa7366467366db627d9f2ee097f",
+                "sha256:cba4cfa1052819d16699e1d55d18c92b6e094d4517c41dd231a8b9f87b6fa681",
+                "sha256:cea7daf9fc7ae6687cf1e2c049752f19f146fdc37c2cc376e7d0032cf4f25347",
+                "sha256:cf6c4150f8c0e32d241436526f3c3f9cbd34429492abddbada2ffcff506c51af",
+                "sha256:d09c333d36c1409d56a9d29b3a1b800a42c76a57a5a8907eacdbce3f18768246",
+                "sha256:d27b84d5880f6d8aa9ae3edb253c59d9f6642ffbb2c889b78b60361eed449787",
+                "sha256:d2ccec9ac47cf4e04897619c0e0c1a48c54a71bdf045117d3a26f80d38ab1fb0",
+                "sha256:d71264a80f3fcf512eb4f18f59423fe82d6e346ee97b90625f283df56aee103f",
+                "sha256:d93f3307ad32a27bda2e88ec81134b823c240aa3abb55821a8da553eed8d9439",
+                "sha256:d9631c5e8b5b3a0fda99cb0d29c18133bca1e18aea9effe55adb3da1adef80d3",
+                "sha256:ddfab44e4489bd79bda09d84c430677fc7f0a4939a73d2bba3073036f487a0a6",
+                "sha256:e7048abd75fe40712005bcfc06bb44b9dfcd8e101dda2ecf2f5aa46115ad07ca",
+                "sha256:e73091191e4280403bde6c9a52a6999d69cdfde498f1fdf629105247599b57ec",
+                "sha256:e800776a79a5aabdb17dcc2346a7d66d0777e942e4cd251defeb084762ecd17d",
+                "sha256:edc9fc7051e3350643ad929df55c451899bb9ae6d24998a949d2e4c87fb596d3",
+                "sha256:f089118d584e859c62b3da0892b88a83d611c2033ac410e929cb6754eec0ed16",
+                "sha256:f174bbd484294ed9fdf09437f889f95807e5f229d5d93588d34e92106fbf6717",
+                "sha256:f508b0491767bb1f2b87fdfacaba5f7eddc2f867740ec69ece6d1946d29029a6",
+                "sha256:f7a31fc1e1bd362874863fdeed71aed92d348f5336fd84f2197ba40c59f061bd",
+                "sha256:f9479aa06a793c5aeba49ce5c5692ffb51fcd9a7016e017d555d5e2b0045d212"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==0.2.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.2.1"
         },
         "pycparser": {
             "hashes": [
@@ -1118,6 +1123,14 @@
             ],
             "version": "==2024.2"
         },
+        "referencing": {
+            "hashes": [
+                "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c",
+                "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.35.1"
+        },
         "requests": {
             "hashes": [
                 "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
@@ -1126,6 +1139,113 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.32.3"
         },
+        "rpds-py": {
+            "hashes": [
+                "sha256:034964ea0ea09645bdde13038b38abb14be0aa747f20fcfab6181207dd9e0483",
+                "sha256:0686f2c16eafdc2c6b4ce6e86e5b3092e87db09ae64be2787616444eb35b9756",
+                "sha256:0903ffdb5b9007e503203b6285e4ff0faf96d875c19f1d103b475acf7d9f7311",
+                "sha256:1212cb231f2002934cd8d71a0d718fdd9d9a2dd671e0feef8501038df3508026",
+                "sha256:1357c3092702078b7782b6ebd5ba9b22c1a291c34fbf9d8f1a48237466ac7758",
+                "sha256:1a6cc4eb1e86364331928acafb2bb41d8ab735ca3caf2d6019b9f6dac3f4f65d",
+                "sha256:208ce1d8e3af138d1d9b21d7206356b7f29b96675e0113aea652cf024e4ddfdc",
+                "sha256:2498ff422823be087b48bc82710deb87ac34f6b7c8034ee39920647647de1e60",
+                "sha256:24c28df05bd284879d0fac850ba697077d2a33b7ebcaea6318d6b6cdfdc86ddc",
+                "sha256:2a57300cc8b034c5707085249efd09f19116bb80278d0ec925d7f3710165c510",
+                "sha256:2d2fc3ab021be3e0b5aec6d4164f2689d231b8bfc5185cc454314746aa4aee72",
+                "sha256:2f513758e7cda8bc262e80299a8e3395d7ef7f4ae705be62632f229bc6c33208",
+                "sha256:306da3dfa174b489a3fc63b0872e2226a5ddf94c59875a770d72aff945d5ed96",
+                "sha256:326e42f2b49462e05f8527a1311ce98f9f97c484b3e443ec0ea4638bed3aebcf",
+                "sha256:32a0e24cab2daae0503b06666d516e90a080c1a95aff0406b9f03c6489177c4b",
+                "sha256:32de71c393f126d8203e9815557c7ff4d72ed1ad3aa3f52f6c7938413176750a",
+                "sha256:341a07a4b55126bfae68c9bf24220a73d456111e5eb3dcbdab9fd16de2341224",
+                "sha256:38cacf1f378571450576f2c8ce87da6f3fddc59d744de5c12b37acc23285b1e1",
+                "sha256:3b94b074dcce39976db22ea75c7aea8b22d95e6d3b62f76e20e1179a278521d8",
+                "sha256:3dc7c64b56b82428894f056e9ff6e8ee917ff74fc26b65211a33602c2372e928",
+                "sha256:3f7a048ec1ebc991331d709be4884dc318c9eaafa66dcde8be0933ac0e702149",
+                "sha256:41f65a97bf2c4b161c9f8f89bc37058346bec9b36e373c8ad00a16c957bff625",
+                "sha256:48c95997af9314f4034fe5ba2d837399e786586e220835a578d28fe8161e6ae5",
+                "sha256:49e084d47a66027ac72844f9f52f13d347a9a1f05d4f84381b420e47f836a7fd",
+                "sha256:4b5d17d8f5b885ce50e0cda85f99c0719e365e98b587338535fa566a48375afb",
+                "sha256:4c0321bc03a1c513eca1837e3bba948b975bcf3a172aebc197ab3573207f137a",
+                "sha256:4e7c9aa2353eb0b0d845323857197daa036c2ff8624df990b0d886d22a8f665e",
+                "sha256:4fc4824e38c1e91a73bc820e7caacaf19d0acd557465aceef0420ca59489b390",
+                "sha256:54d8f94dec5765a9edc19610fecf0fdf9cab36cbb9def1213188215f735a6f98",
+                "sha256:574c5c94213bc9990805bfd7e4ba3826d3c098516cbc19f0d0ef0433ad93fa06",
+                "sha256:59e63da174ff287db05ef7c21d75974a5bac727ed60452aeb3a14278477842a8",
+                "sha256:5ae7927cd2b869ca4dc645169d8af5494a29c99afd0ea0f24dd00c811ab1d8b8",
+                "sha256:5f21e1278c9456cd601832375c778ca44614d3433996488221a56572c223f04a",
+                "sha256:5fdf91a7c07f40e47b193f2acae0ed9da35d09325d7c3c3279f722b7cbf3d264",
+                "sha256:62ab12fe03ffc49978d29de9c31bbb216610157f7e5ca8e172fed6642aead3be",
+                "sha256:632d2fdddd9fbe3ac8896a119fd18a71fc95ca9c4cbe5223096c142d8c4a2b1d",
+                "sha256:64a0c965a1e299c9b280006bdb15c276c427c45360aed676305dc36bcaa4d13c",
+                "sha256:67e013a17a3db4d98cc228fd5aeb36a51b0f5cf7330b9102a552060f1fe4e560",
+                "sha256:6b639a19e1791b646d27f15d17530a51722cc728d43b2dff3aeb904f92d91bac",
+                "sha256:6b6e4bcfc32f831bfe3d6d8a5acedfbfd5e252a03c83fa24813b277a3a8a13ca",
+                "sha256:7539dbb8f705e13629ba6f23388976aad809e387f32a6e5c0712e4e8d9bfcce7",
+                "sha256:758098b38c344d9a7f279baf0689261777e601f620078ef5afdc9bd3339965c3",
+                "sha256:762206ba3bf1d6c8c9e0055871d3c0d5b074b7c3120193e6c067e7866f106ab1",
+                "sha256:771c9a3851beaa617d8c8115d65f834a2b52490f42ee2b88b13f1fc5529e9e0c",
+                "sha256:81e7a27365b02fe70a77f1365376879917235b3fec551d19b4c91b51d0bc1d07",
+                "sha256:8338db3c76833d02dc21c3e2c42534091341d26e4f7ba32c6032bb558a02e07b",
+                "sha256:8426f97117b914b9bfb2a7bd46edc148e8defda728a55a5df3a564abe70cd7a4",
+                "sha256:842855bbb113a19c393c6de5aa6ed9a26c6b13c2fead5e49114d39f0d08b94d8",
+                "sha256:87453d491369cd8018016d2714a13e8461975161703c18ee31eecf087a8ae5d4",
+                "sha256:875fe8dffb43c20f68379ee098b035a7038d7903c795d46715f66575a7050b19",
+                "sha256:8ad4dfda52e64af3202ceb2143a62deba97894b71c64a4405ee80f6b3ea77285",
+                "sha256:8c48fc7458fe3a74dcdf56ba3534ff41bd421f69436df09ff3497fdaac18b431",
+                "sha256:8cbb040fec8eddd5a6a75e737fd73c9ce37e51f94bacdd0b178d0174a4758395",
+                "sha256:92d28a608127b357da47c99e0d0e0655ca2060286540fe9f2a25a2e8ac666e05",
+                "sha256:931bf3d0705b2834fed29354f35170fa022fe22a95542b61b7c66aca5f8a224f",
+                "sha256:93bbd66f46dddc41e8c656130c97c0fb515e0fa44e1eebb2592769dbbd41b2f5",
+                "sha256:9ad4640a409bc2b7d22b7921e7660f0db96c5c8c69fbb2e8f3261d4f71d33983",
+                "sha256:a4366f264fa60d3c109f0b27af0cd9eb8d46746bd70bd3d9d425f035b6c7e286",
+                "sha256:a73ed43d64209e853bba567a543170267a5cd64f359540b0ca2d597e329ba172",
+                "sha256:a810a57ce5e8ecf8eac6ec4dab534ff80c34e5a2c31db60e992009cd20f58e0f",
+                "sha256:b4660943030406aaa40ec9f51960dd88049903d9536bc3c8ebb5cc4e1f119bbe",
+                "sha256:b8906f537978da3f7f0bd1ba37b69f6a877bb43312023b086582707d2835bf2f",
+                "sha256:b91bfef5daa2a5a4fe62f8d317fc91a626073639f951f851bd2cb252d01bc6c5",
+                "sha256:ba1fc34d0b2f6fd53377a4c954116251eba6d076bf64f903311f4a7d27d10acd",
+                "sha256:ba235e00e0878ba1080b0f2a761f143b2a2d1c354f3d8e507fbf2f3de401bf18",
+                "sha256:bb11809b0de643a292a82f728c494a2bbef0e30a7c42d37464abbd6bef7ca7b1",
+                "sha256:c17b43fe9c6da16885e3fe28922bcd1a029e61631fb771c7d501019b40bcc904",
+                "sha256:c1c21030ed494deb10226f90e2dbd84a012d59810c409832714a3dd576527be2",
+                "sha256:c398a5a8e258dfdc5ea2aa4e5aa2ca3207f654a8eb268693dd1a76939074a588",
+                "sha256:c637188b930175c256f13adbfc427b83ec7e64476d1ec9d6608f312bb84e06c3",
+                "sha256:c7b4450093c0c909299770226fb0285be47b0a57545bae25b5c4e51566b0e587",
+                "sha256:c8fd7a16f7a047e06c747cfcf2acef3ac316132df1c6077445b29ee6f3f3a70b",
+                "sha256:ca505fd3767a09a139737f3278bc8a485cb64043062da89bcba27e2f2ea78d33",
+                "sha256:d1522025cda9e57329aade769f56e5793b2a5da7759a21914ee10e67e17e601e",
+                "sha256:d276280649305c1da6cdd84585d48ae1f0efa67434d8b10d2df95228e59a05bb",
+                "sha256:d33622dc63c295788eed09dbb1d11bed178909d3267b02d873116ee6be368244",
+                "sha256:d4f2af3107fe4dc40c0d1a2409863f5249c6796398a1d83c1d99a0b3fa6cfb8d",
+                "sha256:d5469b347445d1c31105f33e7bfc9a8ba213d48e42641a610dda65bf9e3c83f5",
+                "sha256:d80fd710b3307a3c63809048b72c536689b9b0b31a2518339c3f1a4d29c73d7a",
+                "sha256:d9bb9242b38a664f307b3b897f093896f7ed51ef4fe25a0502e5a368de9151ea",
+                "sha256:d9ceca96df54cb1675a0b7f52f1c6d5d1df62c5b40741ba211780f1b05a282a2",
+                "sha256:dc2c00acdf68f1f69a476b770af311a7dc3955b7de228b04a40bcc51ac4d743b",
+                "sha256:dfdabdf8519c93908b2bf0f87c3f86f9e88bab279fb4acfd0907519ca5a1739f",
+                "sha256:e04919ffa9a728c446b27b6b625fa1d00ece221bdb9d633e978a7e0353a12c0e",
+                "sha256:e0abcce5e874474d3eab5ad53be03dae2abe651d248bdeaabe83708e82969e78",
+                "sha256:e1c04fb380bc8efaae2fdf17ed6cd5d223da78a8b0b18a610f53d4c5d6e31dfd",
+                "sha256:e23dcdd4b2ff9c6b3317ea7921b210d39592f8ca1cdea58ada25b202c65c0a69",
+                "sha256:e34a3e665d38d0749072e6565400c8ce9abae976e338919a0dfbfb0e1ba43068",
+                "sha256:e6da2e0500742e0f157f005924a0589f2e2dcbfdd6cd0cc0abce367433e989be",
+                "sha256:e9aa4af6b879bb75a3c7766fbf49d77f4097dd12b548ecbbd8b3f85caa833281",
+                "sha256:e9bbdba9e75b1a9ee1dd1335034dad998ef1acc08492226c6fd50aa773bdfa7d",
+                "sha256:e9d4293b21c69ee4f9e1a99ac4f772951d345611c614a0cfae2ec6b565279bc9",
+                "sha256:eadd2417e83a77ce3ae4a0efd08cb0ebdfd317b6406d11020354a53ad458ec84",
+                "sha256:ed0102146574e5e9f079b2e1a06e6b5b12a691f9c74a65b93b7f3d4feda566c6",
+                "sha256:f0fb8efc9e579acf1e556fd86277fecec320c21ca9b5d39db96433ad8c45bc4a",
+                "sha256:f4e9946c8c7def17e4fcb5eddb14c4eb6ebc7f6f309075e6c8d23b133c104607",
+                "sha256:f7649c8b8e4bd1ccc5fcbd51a855d57a617deeba19c66e3d04b1abecc61036b2",
+                "sha256:f980a0640599a74f27fd9d50c84c293f1cb7afc2046c5c6d3efaf8ec7cdbc326",
+                "sha256:f9dc2113e0cf0dd637751ca736186fca63664939ceb9f9f67e93ade88c69c0c9",
+                "sha256:fde778947304e55fc732bc8ea5c6063e74244ac1808471cb498983a210aaf62c",
+                "sha256:fe23687924b25a2dee52fab15976fd6577ed8518072bcda9ff2e2b88ab1f168b"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==0.22.0"
+        },
         "six": {
             "hashes": [
                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -1141,22 +1261,6 @@
             ],
             "version": "==2.4.0"
         },
-        "tomli": {
-            "hashes": [
-                "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38",
-                "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"
-            ],
-            "markers": "python_version < '3.11'",
-            "version": "==2.0.2"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
-                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
-            ],
-            "markers": "python_version < '3.10'",
-            "version": "==4.12.2"
-        },
         "tzdata": {
             "hashes": [
                 "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc",
@@ -1221,91 +1325,91 @@
         },
         "yarl": {
             "hashes": [
-                "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac",
-                "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47",
-                "sha256:0b1794853124e2f663f0ea54efb0340b457f08d40a1cef78edfa086576179c91",
-                "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5",
-                "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df",
-                "sha256:14bc88baa44e1f84164a392827b5defb4fa8e56b93fecac3d15315e7c8e5d8b3",
-                "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463",
-                "sha256:16bca6678a83657dd48df84b51bd56a6c6bd401853aef6d09dc2506a78484c7b",
-                "sha256:1a3b91c44efa29e6c8ef8a9a2b583347998e2ba52c5d8280dbd5919c02dfc3b5",
-                "sha256:1a52a1ffdd824fb1835272e125385c32fd8b17fbdefeedcb4d543cc23b332d74",
-                "sha256:1ce36ded585f45b1e9bb36d0ae94765c6608b43bd2e7f5f88079f7a85c61a4d3",
-                "sha256:299f11b44d8d3a588234adbe01112126010bd96d9139c3ba7b3badd9829261c3",
-                "sha256:2b24ec55fad43e476905eceaf14f41f6478780b870eda5d08b4d6de9a60b65b4",
-                "sha256:2d374d70fdc36f5863b84e54775452f68639bc862918602d028f89310a034ab0",
-                "sha256:2d9f0606baaec5dd54cb99667fcf85183a7477f3766fbddbe3f385e7fc253299",
-                "sha256:2e7ba4c9377e48fb7b20dedbd473cbcbc13e72e1826917c185157a137dac9df2",
-                "sha256:2f0a6423295a0d282d00e8701fe763eeefba8037e984ad5de44aa349002562ac",
-                "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61",
-                "sha256:380e6c38ef692b8fd5a0f6d1fa8774d81ebc08cfbd624b1bca62a4d4af2f9931",
-                "sha256:3b74ff4767d3ef47ffe0cd1d89379dc4d828d4873e5528976ced3b44fe5b0a21",
-                "sha256:3e844be8d536afa129366d9af76ed7cb8dfefec99f5f1c9e4f8ae542279a6dc3",
-                "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7",
-                "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96",
-                "sha256:482c122b72e3c5ec98f11457aeb436ae4aecca75de19b3d1de7cf88bc40db82f",
-                "sha256:561c87fea99545ef7d692403c110b2f99dced6dff93056d6e04384ad3bc46243",
-                "sha256:578d00c9b7fccfa1745a44f4eddfdc99d723d157dad26764538fbdda37209857",
-                "sha256:58c8e9620eb82a189c6c40cb6b59b4e35b2ee68b1f2afa6597732a2b467d7e8f",
-                "sha256:5b29beab10211a746f9846baa39275e80034e065460d99eb51e45c9a9495bcca",
-                "sha256:5d1d42556b063d579cae59e37a38c61f4402b47d70c29f0ef15cee1acaa64488",
-                "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da",
-                "sha256:62a91aefff3d11bf60e5956d340eb507a983a7ec802b19072bb989ce120cd948",
-                "sha256:64cc6e97f14cf8a275d79c5002281f3040c12e2e4220623b5759ea7f9868d6a5",
-                "sha256:6f4c9156c4d1eb490fe374fb294deeb7bc7eaccda50e23775b2354b6a6739934",
-                "sha256:7294e38f9aa2e9f05f765b28ffdc5d81378508ce6dadbe93f6d464a8c9594473",
-                "sha256:7615058aabad54416ddac99ade09a5510cf77039a3b903e94e8922f25ed203d7",
-                "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685",
-                "sha256:7f63d176a81555984e91f2c84c2a574a61cab7111cc907e176f0f01538e9ff6e",
-                "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147",
-                "sha256:7fac95714b09da9278a0b52e492466f773cfe37651cf467a83a1b659be24bf71",
-                "sha256:81713b70bea5c1386dc2f32a8f0dab4148a2928c7495c808c541ee0aae614d67",
-                "sha256:846dd2e1243407133d3195d2d7e4ceefcaa5f5bf7278f0a9bda00967e6326b04",
-                "sha256:84c063af19ef5130084db70ada40ce63a84f6c1ef4d3dbc34e5e8c4febb20822",
-                "sha256:881764d610e3269964fc4bb3c19bb6fce55422828e152b885609ec176b41cf11",
-                "sha256:8994b29c462de9a8fce2d591028b986dbbe1b32f3ad600b2d3e1c482c93abad6",
-                "sha256:8c79e9d7e3d8a32d4824250a9c6401194fb4c2ad9a0cec8f6a96e09a582c2cc0",
-                "sha256:8ee427208c675f1b6e344a1f89376a9613fc30b52646a04ac0c1f6587c7e46ec",
-                "sha256:949681f68e0e3c25377462be4b658500e85ca24323d9619fdc41f68d46a1ffda",
-                "sha256:9e275792097c9f7e80741c36de3b61917aebecc08a67ae62899b074566ff8556",
-                "sha256:9fb815155aac6bfa8d86184079652c9715c812d506b22cfa369196ef4e99d1b4",
-                "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c",
-                "sha256:a7ac5b4984c468ce4f4a553df281450df0a34aefae02e58d77a0847be8d1e11f",
-                "sha256:aa46dce75078fceaf7cecac5817422febb4355fbdda440db55206e3bd288cfb8",
-                "sha256:ae3476e934b9d714aa8000d2e4c01eb2590eee10b9d8cd03e7983ad65dfbfcba",
-                "sha256:b0341e6d9a0c0e3cdc65857ef518bb05b410dbd70d749a0d33ac0f39e81a4258",
-                "sha256:b40d1bf6e6f74f7c0a567a9e5e778bbd4699d1d3d2c0fe46f4b717eef9e96b95",
-                "sha256:b5c4804e4039f487e942c13381e6c27b4b4e66066d94ef1fae3f6ba8b953f383",
-                "sha256:b5d6a6c9602fd4598fa07e0389e19fe199ae96449008d8304bf5d47cb745462e",
-                "sha256:b5f1ac7359e17efe0b6e5fec21de34145caef22b260e978336f325d5c84e6938",
-                "sha256:c0167540094838ee9093ef6cc2c69d0074bbf84a432b4995835e8e5a0d984374",
-                "sha256:c180ac742a083e109c1a18151f4dd8675f32679985a1c750d2ff806796165b55",
-                "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139",
-                "sha256:c7e177c619342e407415d4f35dec63d2d134d951e24b5166afcdfd1362828e17",
-                "sha256:cbad927ea8ed814622305d842c93412cb47bd39a496ed0f96bfd42b922b4a217",
-                "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d",
-                "sha256:cc7c92c1baa629cb03ecb0c3d12564f172218fb1739f54bf5f3881844daadc6d",
-                "sha256:cc7d768260f4ba4ea01741c1b5fe3d3a6c70eb91c87f4c8761bbcce5181beafe",
-                "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199",
-                "sha256:d216e5d9b8749563c7f2c6f7a0831057ec844c68b4c11cb10fc62d4fd373c26d",
-                "sha256:d401f07261dc5aa36c2e4efc308548f6ae943bfff20fcadb0a07517a26b196d8",
-                "sha256:d6324274b4e0e2fa1b3eccb25997b1c9ed134ff61d296448ab8269f5ac068c4c",
-                "sha256:d8a8b74d843c2638f3864a17d97a4acda58e40d3e44b6303b8cc3d3c44ae2d29",
-                "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172",
-                "sha256:de599af166970d6a61accde358ec9ded821234cbbc8c6413acfec06056b8e860",
-                "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7",
-                "sha256:e5b078134f48552c4d9527db2f7da0b5359abd49393cdf9794017baec7506170",
-                "sha256:eb6dce402734575e1a8cc0bb1509afca508a400a57ce13d306ea2c663bad1138",
-                "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06",
-                "sha256:f5efe0661b9fcd6246f27957f6ae1c0eb29bc60552820f01e970b4996e016004",
-                "sha256:f9cbfbc5faca235fbdf531b93aa0f9f005ec7d267d9d738761a4d42b744ea159",
-                "sha256:fbea1751729afe607d84acfd01efd95e3b31db148a181a441984ce9b3d3469da",
-                "sha256:fca4b4307ebe9c3ec77a084da3a9d1999d164693d16492ca2b64594340999988",
-                "sha256:ff5c6771c7e3511a06555afa317879b7db8d640137ba55d6ab0d0c50425cab75"
+                "sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba",
+                "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193",
+                "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318",
+                "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee",
+                "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e",
+                "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1",
+                "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a",
+                "sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186",
+                "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1",
+                "sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50",
+                "sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640",
+                "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb",
+                "sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8",
+                "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc",
+                "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5",
+                "sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58",
+                "sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2",
+                "sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393",
+                "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24",
+                "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b",
+                "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910",
+                "sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c",
+                "sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272",
+                "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed",
+                "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1",
+                "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04",
+                "sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d",
+                "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5",
+                "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d",
+                "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889",
+                "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae",
+                "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b",
+                "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c",
+                "sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576",
+                "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34",
+                "sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477",
+                "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990",
+                "sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2",
+                "sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512",
+                "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069",
+                "sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a",
+                "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6",
+                "sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0",
+                "sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8",
+                "sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb",
+                "sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa",
+                "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8",
+                "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e",
+                "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e",
+                "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985",
+                "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8",
+                "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1",
+                "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5",
+                "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690",
+                "sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10",
+                "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789",
+                "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b",
+                "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca",
+                "sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e",
+                "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5",
+                "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59",
+                "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9",
+                "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8",
+                "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db",
+                "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde",
+                "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7",
+                "sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb",
+                "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3",
+                "sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6",
+                "sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285",
+                "sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb",
+                "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8",
+                "sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482",
+                "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd",
+                "sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75",
+                "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760",
+                "sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782",
+                "sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53",
+                "sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2",
+                "sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1",
+                "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719",
+                "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"
             ],
             "markers": "python_version >= '3.9'",
-            "version": "==1.17.1"
+            "version": "==1.18.3"
         }
     },
     "develop": {
diff --git a/requirements.txt b/requirements.txt
index 2a949453..800bd3bc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,7 @@ defusedxml==0.7.1
 frozendict==2.3.8
 globus-sdk==1.6.1
 hypothesis==6.81.1
+jsonschema==4.23.0
 openpyxl==3.1.2
 pandas==2.2.3
 pytest-aiohttp==1.0.4
diff --git a/staging_service/app.py b/staging_service/app.py
index c87608ea..8588b7ba 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -4,6 +4,7 @@
 import shutil
 import sys
 from collections import defaultdict
+from collections.abc import Callable
 from pathlib import Path as PathPy
 from urllib.parse import parse_qs, unquote
 
@@ -107,8 +108,18 @@ def _file_type_resolver(path: PathPy) -> FileTypeResolution:
             ext = path.name
         return FileTypeResolution(unsupported_type=ext)
 
-def _dts_file_resolver(_: PathPy) -> FileTypeResolution:
-    return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, _DTS_MANIFEST_SCHEMA))
+def _make_dts_file_resolver() -> Callable[[Path], FileTypeResolution]:
+    """Makes a DTS file resolver.
+
+    This looks a little goofy, but it ensures that the DTS manifest schema file
+    only gets loaded once per API call, no matter how many DTS manifest files are
+    expected to be parsed. It also prevents having it stick around in memory.
+    """
+    with open(_DTS_MANIFEST_SCHEMA) as schema_file:
+        dts_schema = json.load(schema_file)
+    def dts_file_resolver(_: PathPy):
+        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, dts_schema))
+    return dts_file_resolver
 
 @routes.get("/bulk_specification/{query:.*}")
 async def bulk_specification(request: web.Request) -> web.json_response:
@@ -133,11 +144,10 @@ async def bulk_specification(request: web.Request) -> web.json_response:
         paths[PathPy(p.full_path)] = PathPy(p.user_path)
     as_dts = params.get("dts", ["0"])[0] == "1"
 
-
     # list(dict) returns a list of the dict keys in insertion order (py3.7+)
     file_type_resolver = _file_type_resolver
     if as_dts:
-        file_type_resolver = _dts_file_resolver
+        file_type_resolver = _make_dts_file_resolver()
     res = parse_import_specifications(
         tuple(list(paths)),
         file_type_resolver,
diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index 73e78c46..cb9c266c 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -4,6 +4,7 @@
 
 import csv
 import json
+import jsonschema
 import math
 import re
 from pathlib import Path
@@ -319,7 +320,7 @@ def parse_excel(path: Path) -> ParseResults:
         return _error(Error(ErrorType.PARSE_FAIL, "No non-header data in file", spcsrc))
 
 
-def parse_dts_manifest(path: Path, dts_manifest_schema: Path) -> ParseResults:
+def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
     """
     Parse the provided DTS manifest file. Expected to be JSON, and will fail otherwise.
     The manifest should have this format, with expected keys included:
@@ -352,8 +353,16 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: Path) -> ParseResults:
         with open(path, "r") as manifest:
             manifest_json = json.load(manifest)
         if not isinstance(manifest_json, dict):
-            errors.append(Error(ErrorType.PARSE_FAIL, "Manifest is not a dictionary", spcsrc))
-
+            return _error(Error(ErrorType.PARSE_FAIL, "Manifest is not a dictionary", spcsrc))
+        validator = jsonschema.Draft202012Validator(dts_manifest_schema)
+        for err in validator.iter_errors(manifest_json):
+            err_str = err.message
+            err_path = err.absolute_path
+            if err_path:
+                if isinstance(err_path[-1], int):
+                    err_path[-1] = f"item {err_path[-1]}"
+                err_str += f" at {'/'.join(err_path)}"
+            errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
     except json.JSONDecodeError:
         return _error(Error(ErrorType.PARSE_FAIL, "File must be in JSON format", spcsrc))
     except FileNotFoundError:
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 61214cfd..79ae7e00 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -1,13 +1,13 @@
 import json
 import os
+import pytest
 import uuid
 from collections.abc import Callable, Generator
 from pathlib import Path
 
 # TODO update to C impl when fixed: https://github.com/Marco-Sulla/python-frozendict/issues/26
 from frozendict import frozendict
-from pytest import fixture
-
+from staging_service.app import inject_config_dependencies
 from staging_service.import_specifications.individual_parsers import (
     Error,
     ErrorType,
@@ -20,11 +20,11 @@
     parse_tsv,
 )
 from tests.test_app import FileUtil
+from tests.test_utils import bootstrap_config
 
 _TEST_DATA_DIR = (Path(__file__).parent / "test_data").resolve()
 
-
-@fixture(scope="module", name="temp_dir")
+@pytest.fixture(scope="module", name="temp_dir")
 def temp_dir_fixture() -> Generator[Path, None, None]:
     with FileUtil() as fu:
         childdir = Path(fu.make_dir(str(uuid.uuid4()))).resolve()
@@ -33,6 +33,12 @@ def temp_dir_fixture() -> Generator[Path, None, None]:
 
     # FileUtil will auto delete after exiting
 
+@pytest.fixture(scope="module")
+def dts_schema() -> Generator[dict, None, None]:
+    config = bootstrap_config()
+    with open(config["staging_service"]["DTS_MANIFEST_SCHEMA"]) as dts_schema_file:
+        schema = json.load(dts_schema_file)
+        yield schema
 
 ##########################################
 # xSV tests
@@ -769,21 +775,21 @@ def test_excel_parse_fail_unequal_rows():
     )
 
 
-def test_dts_manifest_parse_success():
+def test_dts_manifest_parse_success(dts_schema):
     f = _get_test_file("manifest_small.json")
-    res = parse_dts_manifest(f)
+    res = parse_dts_manifest(f, dts_schema)
     # fails for now
     assert res.results is None
     assert res.errors == tuple(
         [
             Error(
-                ErrorType.PARSE_FAIL, "No import specification data in file", SpecificationSource(f)
+                ErrorType.PARSE_FAIL, "'instructions' is a required property", SpecificationSource(f)
             )
         ]
     )
 
 
-@fixture(scope="module")
+@pytest.fixture(scope="module")
 def write_dts_manifest(temp_dir: Generator[Path, None, None]) -> Callable[[dict | list], Path]:
     def manifest_writer(input_json: dict | list) -> Path:
         file_path = temp_dir / str(uuid.uuid4())
@@ -793,25 +799,25 @@ def manifest_writer(input_json: dict | list) -> Path:
 
     return manifest_writer
 
-
-def _dts_manifest_parse_fail(input_file: Path, errors: list[Error]):
+def _dts_manifest_parse_fail(input_file: Path, schema: dict, errors: list[Error]):
     """
     Tests a failing DTS manifest parse.
     input_file - the path to the input file. Might be a directory or not exist.
     errors - a list of Error objects expected to be in the order generated by
         the call to parse_dts_manifest.
     """
-    res = parse_dts_manifest(input_file)
+    res = parse_dts_manifest(input_file, schema)
     assert res.results is None
     assert res.errors == tuple(errors)
 
 
-def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None]):
+def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
     test_file = temp_dir / str(uuid.uuid4())
     with open(test_file, "w", encoding="utf-8") as outfile:
         outfile.write("totally not json")
     _dts_manifest_parse_fail(
         test_file,
+        dts_schema,
         [
             Error(
                 ErrorType.PARSE_FAIL, "File must be in JSON format", SpecificationSource(test_file)
@@ -820,10 +826,11 @@ def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None]):
     )
 
 
-def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path]):
-    manifest_path = write_dts_manifest(["wrong_format"])
+def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path], dts_schema: Generator[Path, None, None]):
+    manifest_path = write_dts_manifest(["wrong_format", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
     _dts_manifest_parse_fail(
         manifest_path,
+        dts_schema,
         [
             Error(
                 ErrorType.PARSE_FAIL,
@@ -834,19 +841,21 @@ def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path]
     )
 
 
-def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None]):
+def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
     manifest_path = temp_dir / "not_a_file"
     _dts_manifest_parse_fail(
         manifest_path,
+        dts_schema,
         [Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(manifest_path))],
     )
 
 
-def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None]):
+def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
     test_file = temp_dir / "testdir.json"
     os.makedirs(test_file, exist_ok=True)
     _dts_manifest_parse_fail(
         test_file,
+        dts_schema,
         [
             Error(
                 ErrorType.PARSE_FAIL,
@@ -855,3 +864,8 @@ def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None]):
             )
         ],
     )
+
+
+def test_dts_manifest_bad_schema():
+    f = _get_test_file("manifest_small.json")
+

From 3bc8c890706760c957f062f87d77f9d34851e66d Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 3 Dec 2024 13:57:17 -0500
Subject: [PATCH 04/22] add checks for bad schema

---
 .../import_specifications/individual_parsers.py           | 4 ++++
 tests/import_specifications/test_individual_parsers.py    | 8 +++++---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index cb9c266c..cde8cf83 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -10,6 +10,7 @@
 from pathlib import Path
 from typing import Any, Optional, Tuple, Union
 
+import jsonschema.exceptions
 import magic
 import pandas
 from frozendict import frozendict
@@ -350,6 +351,7 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
     # dummy for now
     results = {}
     try:
+        jsonschema.Draft202012Validator.check_schema(dts_manifest_schema)
         with open(path, "r") as manifest:
             manifest_json = json.load(manifest)
         if not isinstance(manifest_json, dict):
@@ -363,6 +365,8 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
                     err_path[-1] = f"item {err_path[-1]}"
                 err_str += f" at {'/'.join(err_path)}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
+    except jsonschema.exceptions.SchemaError as err:
+        return _error(Error(ErrorType.OTHER, "Manifest schema is invalid", spcsrc))
     except json.JSONDecodeError:
         return _error(Error(ErrorType.PARSE_FAIL, "File must be in JSON format", spcsrc))
     except FileNotFoundError:
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 79ae7e00..947af35c 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -865,7 +865,9 @@ def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], d
         ],
     )
 
-
-def test_dts_manifest_bad_schema():
+@pytest.mark.parametrize("bad_schema", [None, 1, [], {"foo"}])
+def test_dts_manifest_bad_schema(bad_schema):
     f = _get_test_file("manifest_small.json")
-
+    _dts_manifest_parse_fail(
+        f, bad_schema, [Error(ErrorType.OTHER, "Manifest schema is invalid", SpecificationSource(f))]
+    )

From 8251fffd96b72d17c9e9063a31de7d2c5e35970e Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 3 Dec 2024 15:59:34 -0500
Subject: [PATCH 05/22] update tests

---
 .../test_individual_parsers.py                | 64 +++++++++++++++++--
 1 file changed, 57 insertions(+), 7 deletions(-)

diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 947af35c..c6d1961b 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -4,6 +4,7 @@
 import uuid
 from collections.abc import Callable, Generator
 from pathlib import Path
+from typing import Any
 
 # TODO update to C impl when fixed: https://github.com/Marco-Sulla/python-frozendict/issues/26
 from frozendict import frozendict
@@ -34,7 +35,7 @@ def temp_dir_fixture() -> Generator[Path, None, None]:
     # FileUtil will auto delete after exiting
 
 @pytest.fixture(scope="module")
-def dts_schema() -> Generator[dict, None, None]:
+def dts_schema() -> Generator[dict[str, Any], None, None]:
     config = bootstrap_config()
     with open(config["staging_service"]["DTS_MANIFEST_SCHEMA"]) as dts_schema_file:
         schema = json.load(dts_schema_file)
@@ -775,7 +776,7 @@ def test_excel_parse_fail_unequal_rows():
     )
 
 
-def test_dts_manifest_parse_success(dts_schema):
+def test_dts_manifest_parse_success(dts_schema: dict[str, Any]):
     f = _get_test_file("manifest_small.json")
     res = parse_dts_manifest(f, dts_schema)
     # fails for now
@@ -811,7 +812,7 @@ def _dts_manifest_parse_fail(input_file: Path, schema: dict, errors: list[Error]
     assert res.errors == tuple(errors)
 
 
-def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
+def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
     test_file = temp_dir / str(uuid.uuid4())
     with open(test_file, "w", encoding="utf-8") as outfile:
         outfile.write("totally not json")
@@ -826,8 +827,8 @@ def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema
     )
 
 
-def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path], dts_schema: Generator[Path, None, None]):
-    manifest_path = write_dts_manifest(["wrong_format", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
+def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
+    manifest_path = write_dts_manifest(["wrong_format"])
     _dts_manifest_parse_fail(
         manifest_path,
         dts_schema,
@@ -841,7 +842,7 @@ def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path]
     )
 
 
-def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
+def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
     manifest_path = temp_dir / "not_a_file"
     _dts_manifest_parse_fail(
         manifest_path,
@@ -850,7 +851,7 @@ def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schem
     )
 
 
-def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], dts_schema: Generator[Path, None, None]):
+def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
     test_file = temp_dir / "testdir.json"
     os.makedirs(test_file, exist_ok=True)
     _dts_manifest_parse_fail(
@@ -865,9 +866,58 @@ def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], d
         ],
     )
 
+
 @pytest.mark.parametrize("bad_schema", [None, 1, [], {"foo"}])
 def test_dts_manifest_bad_schema(bad_schema):
     f = _get_test_file("manifest_small.json")
     _dts_manifest_parse_fail(
         f, bad_schema, [Error(ErrorType.OTHER, "Manifest schema is invalid", SpecificationSource(f))]
     )
+
+
+def test_dts_manifest_no_top_level_keys(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
+    manifest_path = write_dts_manifest({ "missing": "stuff" })
+    _dts_manifest_parse_fail(
+        manifest_path,
+        dts_schema,
+        [
+            Error(
+                ErrorType.PARSE_FAIL,
+                "'resources' is a required property",
+                SpecificationSource(manifest_path)
+            ),
+            Error(
+                ErrorType.PARSE_FAIL,
+                "'instructions' is a required property",
+                SpecificationSource(manifest_path)
+            ),
+        ],
+    )
+
+def test_dts_manifest_fail_with_path(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
+    manifest_path = write_dts_manifest({
+        "resources": [],
+        "instructions": {
+            "protocol": "KBase narrative import",
+            "objects": [
+                {
+                    "data_type": "foo",
+                    "parameters": {}
+                },
+                {
+                    "parameters": {}
+                }
+            ]
+        }
+    })
+    _dts_manifest_parse_fail(
+        manifest_path,
+        dts_schema,
+        [
+            Error(
+                ErrorType.PARSE_FAIL,
+                "'data_type' is a required property at instructions/objects/item 1",
+                SpecificationSource(manifest_path)
+            )
+        ]
+    )

From ab1170668d49d2fc631abde1a070ea4863c21525 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 3 Dec 2024 16:58:38 -0500
Subject: [PATCH 06/22] ruff formatting

---
 staging_service/app.py                        |  4 ++
 staging_service/utils.py                      |  1 +
 .../test_individual_parsers.py                | 61 +++++++++++--------
 3 files changed, 41 insertions(+), 25 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index d0863eca..071b8bf8 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -105,6 +105,7 @@ def _file_type_resolver(path: PathPy) -> FileTypeResolution:
             ext = path.name
         return FileTypeResolution(unsupported_type=ext)
 
+
 def _make_dts_file_resolver() -> Callable[[Path], FileTypeResolution]:
     """Makes a DTS file resolver.
 
@@ -114,10 +115,13 @@ def _make_dts_file_resolver() -> Callable[[Path], FileTypeResolution]:
     """
     with open(_DTS_MANIFEST_SCHEMA) as schema_file:
         dts_schema = json.load(schema_file)
+
     def dts_file_resolver(_: PathPy):
         return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, dts_schema))
+
     return dts_file_resolver
 
+
 @routes.get("/bulk_specification/{query:.*}")
 async def bulk_specification(request: web.Request) -> web.json_response:
     """
diff --git a/staging_service/utils.py b/staging_service/utils.py
index 379fb929..748edcce 100644
--- a/staging_service/utils.py
+++ b/staging_service/utils.py
@@ -86,6 +86,7 @@ def from_full_path(full_path: str):
         jgi_metadata = os.path.join(os.path.dirname(full_path), "." + name + ".jgi")
         return Path(full_path, metadata_path, user_path, name, jgi_metadata)
 
+
 class AclManager:
     def __init__(self):
         """
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index c6d1961b..a9fa0779 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -25,6 +25,7 @@
 
 _TEST_DATA_DIR = (Path(__file__).parent / "test_data").resolve()
 
+
 @pytest.fixture(scope="module", name="temp_dir")
 def temp_dir_fixture() -> Generator[Path, None, None]:
     with FileUtil() as fu:
@@ -34,6 +35,7 @@ def temp_dir_fixture() -> Generator[Path, None, None]:
 
     # FileUtil will auto delete after exiting
 
+
 @pytest.fixture(scope="module")
 def dts_schema() -> Generator[dict[str, Any], None, None]:
     config = bootstrap_config()
@@ -41,6 +43,7 @@ def dts_schema() -> Generator[dict[str, Any], None, None]:
         schema = json.load(dts_schema_file)
         yield schema
 
+
 ##########################################
 # xSV tests
 ##########################################
@@ -784,7 +787,9 @@ def test_dts_manifest_parse_success(dts_schema: dict[str, Any]):
     assert res.errors == tuple(
         [
             Error(
-                ErrorType.PARSE_FAIL, "'instructions' is a required property", SpecificationSource(f)
+                ErrorType.PARSE_FAIL,
+                "'instructions' is a required property",
+                SpecificationSource(f),
             )
         ]
     )
@@ -800,6 +805,7 @@ def manifest_writer(input_json: dict | list) -> Path:
 
     return manifest_writer
 
+
 def _dts_manifest_parse_fail(input_file: Path, schema: dict, errors: list[Error]):
     """
     Tests a failing DTS manifest parse.
@@ -827,7 +833,9 @@ def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema
     )
 
 
-def test_dts_manifest_non_dict(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
+def test_dts_manifest_non_dict(
+    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+):
     manifest_path = write_dts_manifest(["wrong_format"])
     _dts_manifest_parse_fail(
         manifest_path,
@@ -851,7 +859,9 @@ def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schem
     )
 
 
-def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
+def test_dts_manifest_file_is_directory(
+    temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]
+):
     test_file = temp_dir / "testdir.json"
     os.makedirs(test_file, exist_ok=True)
     _dts_manifest_parse_fail(
@@ -871,12 +881,16 @@ def test_dts_manifest_file_is_directory(temp_dir: Generator[Path, None, None], d
 def test_dts_manifest_bad_schema(bad_schema):
     f = _get_test_file("manifest_small.json")
     _dts_manifest_parse_fail(
-        f, bad_schema, [Error(ErrorType.OTHER, "Manifest schema is invalid", SpecificationSource(f))]
+        f,
+        bad_schema,
+        [Error(ErrorType.OTHER, "Manifest schema is invalid", SpecificationSource(f))],
     )
 
 
-def test_dts_manifest_no_top_level_keys(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
-    manifest_path = write_dts_manifest({ "missing": "stuff" })
+def test_dts_manifest_no_top_level_keys(
+    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+):
+    manifest_path = write_dts_manifest({"missing": "stuff"})
     _dts_manifest_parse_fail(
         manifest_path,
         dts_schema,
@@ -884,32 +898,29 @@ def test_dts_manifest_no_top_level_keys(write_dts_manifest: Callable[[dict | lis
             Error(
                 ErrorType.PARSE_FAIL,
                 "'resources' is a required property",
-                SpecificationSource(manifest_path)
+                SpecificationSource(manifest_path),
             ),
             Error(
                 ErrorType.PARSE_FAIL,
                 "'instructions' is a required property",
-                SpecificationSource(manifest_path)
+                SpecificationSource(manifest_path),
             ),
         ],
     )
 
-def test_dts_manifest_fail_with_path(write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]):
-    manifest_path = write_dts_manifest({
-        "resources": [],
-        "instructions": {
-            "protocol": "KBase narrative import",
-            "objects": [
-                {
-                    "data_type": "foo",
-                    "parameters": {}
-                },
-                {
-                    "parameters": {}
-                }
-            ]
+
+def test_dts_manifest_fail_with_path(
+    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+):
+    manifest_path = write_dts_manifest(
+        {
+            "resources": [],
+            "instructions": {
+                "protocol": "KBase narrative import",
+                "objects": [{"data_type": "foo", "parameters": {}}, {"parameters": {}}],
+            },
         }
-    })
+    )
     _dts_manifest_parse_fail(
         manifest_path,
         dts_schema,
@@ -917,7 +928,7 @@ def test_dts_manifest_fail_with_path(write_dts_manifest: Callable[[dict | list],
             Error(
                 ErrorType.PARSE_FAIL,
                 "'data_type' is a required property at instructions/objects/item 1",
-                SpecificationSource(manifest_path)
+                SpecificationSource(manifest_path),
             )
-        ]
+        ],
     )

From bcb5ce3fc9123ffa8a04188c91976226354ab82a Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 3 Dec 2024 17:03:12 -0500
Subject: [PATCH 07/22] ruff check fixes

---
 staging_service/app.py                                 | 1 -
 tests/import_specifications/test_individual_parsers.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index 071b8bf8..ecfb0354 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -5,7 +5,6 @@
 import shutil
 import sys
 from collections import defaultdict
-from collections.abc import Callable
 from pathlib import Path as PathPy
 from urllib.parse import parse_qs, unquote
 
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index a9fa0779..4ef05d8b 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -8,7 +8,6 @@
 
 # TODO update to C impl when fixed: https://github.com/Marco-Sulla/python-frozendict/issues/26
 from frozendict import frozendict
-from staging_service.app import inject_config_dependencies
 from staging_service.import_specifications.individual_parsers import (
     Error,
     ErrorType,

From 8e3a749248076ca09307f531a78ddac171641544 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 4 Dec 2024 13:59:28 -0500
Subject: [PATCH 08/22] remove unused variable

---
 staging_service/import_specifications/individual_parsers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index cde8cf83..3b1dde5d 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -365,7 +365,7 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
                     err_path[-1] = f"item {err_path[-1]}"
                 err_str += f" at {'/'.join(err_path)}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
-    except jsonschema.exceptions.SchemaError as err:
+    except jsonschema.exceptions.SchemaError:
         return _error(Error(ErrorType.OTHER, "Manifest schema is invalid", spcsrc))
     except json.JSONDecodeError:
         return _error(Error(ErrorType.PARSE_FAIL, "File must be in JSON format", spcsrc))

From 37a73c15bc3bbee6abe338376cd1aefa78ebf6b4 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Mon, 9 Dec 2024 20:51:56 -0500
Subject: [PATCH 09/22] fix commit error

---
 staging_service/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index 0a559480..e607641f 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -124,7 +124,7 @@ def dts_file_resolver(path: PathPy) -> FileTypeResolution:
         suffix = path.suffix[1:] if path.suffix else NO_EXTENSION
         if suffix.lower() != JSON_EXTENSION:
             return FileTypeResolution(unsupported_type=suffix)
-        return FileTypeResolution(parser=parse_dts_manifest)
+        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, dts_schema))
 
     return dts_file_resolver
 

From 90b209dede7657cc68a4b0ac7c86a912fe56213d Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Mon, 9 Dec 2024 21:13:59 -0500
Subject: [PATCH 10/22] update jsonschema validation error handling

---
 .../import_specifications/individual_parsers.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index 3b1dde5d..c3f4b0c5 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -361,9 +361,20 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
             err_str = err.message
             err_path = err.absolute_path
             if err_path:
-                if isinstance(err_path[-1], int):
-                    err_path[-1] = f"item {err_path[-1]}"
-                err_str += f" at {'/'.join(err_path)}"
+                # paths can look like, say, ["instructions", "objects", 0, "data_type"]
+                # convert that '0' to "item 0" to be slightly more readable to users.
+                # kind of a mouthful below, but does that conversion in place
+                # The error above gets translated into:
+                # "<err.message> at instructions/objects/item 0/data_type"
+                #
+                # Another example would be if the path is just ["instructions"] and it's missing
+                # a property, the error might be:
+                # "missing property 'foo' for 'instructions'"
+                err_path = [f"item {elem}" if isinstance(elem, int) else elem for elem in err_path]
+                prep = "for"
+                if len(err_path) > 1:
+                    prep = "at"
+                err_str += f" {prep} {'/'.join(err_path)}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
     except jsonschema.exceptions.SchemaError:
         return _error(Error(ErrorType.OTHER, "Manifest schema is invalid", spcsrc))

From 748380c056913268fa72481c2ad35b7263f64807 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Tue, 10 Dec 2024 08:57:46 -0500
Subject: [PATCH 11/22] update schema

---
 import_specifications/schema/dts_manifest.json | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/import_specifications/schema/dts_manifest.json b/import_specifications/schema/dts_manifest.json
index fc70d7ef..07578596 100644
--- a/import_specifications/schema/dts_manifest.json
+++ b/import_specifications/schema/dts_manifest.json
@@ -13,7 +13,20 @@
                         "type": "object",
                         "properties": {
                             "data_type": {"type": "string"},
-                            "parameters": {"type": "object"}
+                            "parameters": {
+                                "type": "object",
+                                "patternProperties": {
+                                    ".*": {
+                                        "oneOf": [
+                                            {"type": "string"},
+                                            {"type": "number"},
+                                            {"type": "boolean"},
+                                            {"type": "null"}
+                                        ]
+                                    }
+                                },
+                                "additionalProperties": false
+                            }
                         },
                         "required": ["data_type", "parameters"]
                     }

From 528bb40edd3e6d4af1412449a855861dbaa22355 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 15:56:08 -0500
Subject: [PATCH 12/22] remove fancy error texts

---
 .../schema/dts_manifest.json                  | 24 +++++++++----------
 .../individual_parsers.py                     | 17 ++-----------
 .../test_individual_parsers.py                |  2 +-
 3 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/import_specifications/schema/dts_manifest.json b/import_specifications/schema/dts_manifest.json
index 07578596..1bf6dc80 100644
--- a/import_specifications/schema/dts_manifest.json
+++ b/import_specifications/schema/dts_manifest.json
@@ -6,7 +6,10 @@
         "instructions": {
             "type": "object",
             "properties": {
-                "protocol": {"type": "string"},
+                "protocol": {
+                    "type": "string",
+                    "const": "KBase narrative import"
+                },
                 "objects": {
                     "type": "array",
                     "items": {
@@ -15,17 +18,14 @@
                             "data_type": {"type": "string"},
                             "parameters": {
                                 "type": "object",
-                                "patternProperties": {
-                                    ".*": {
-                                        "oneOf": [
-                                            {"type": "string"},
-                                            {"type": "number"},
-                                            {"type": "boolean"},
-                                            {"type": "null"}
-                                        ]
-                                    }
-                                },
-                                "additionalProperties": false
+                                "additionalProperties": {
+                                    "oneOf": [
+                                        {"type": "string"},
+                                        {"type": "number"},
+                                        {"type": "boolean"},
+                                        {"type": "null"}
+                                    ]
+                                }
                             }
                         },
                         "required": ["data_type", "parameters"]
diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index c3f4b0c5..f5ca0b64 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -359,22 +359,9 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
         validator = jsonschema.Draft202012Validator(dts_manifest_schema)
         for err in validator.iter_errors(manifest_json):
             err_str = err.message
-            err_path = err.absolute_path
+            err_path = list(err.absolute_path)
             if err_path:
-                # paths can look like, say, ["instructions", "objects", 0, "data_type"]
-                # convert that '0' to "item 0" to be slightly more readable to users.
-                # kind of a mouthful below, but does that conversion in place
-                # The error above gets translated into:
-                # "<err.message> at instructions/objects/item 0/data_type"
-                #
-                # Another example would be if the path is just ["instructions"] and it's missing
-                # a property, the error might be:
-                # "missing property 'foo' for 'instructions'"
-                err_path = [f"item {elem}" if isinstance(elem, int) else elem for elem in err_path]
-                prep = "for"
-                if len(err_path) > 1:
-                    prep = "at"
-                err_str += f" {prep} {'/'.join(err_path)}"
+                err_str += f" at {err_path}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
     except jsonschema.exceptions.SchemaError:
         return _error(Error(ErrorType.OTHER, "Manifest schema is invalid", spcsrc))
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 4ef05d8b..87a63d41 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -926,7 +926,7 @@ def test_dts_manifest_fail_with_path(
         [
             Error(
                 ErrorType.PARSE_FAIL,
-                "'data_type' is a required property at instructions/objects/item 1",
+                "'data_type' is a required property at ['instructions', 'objects', 1]",
                 SpecificationSource(manifest_path),
             )
         ],

From 86c77ad3ae2842682cf3c1081fe5d6e9a4307143 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 16:31:28 -0500
Subject: [PATCH 13/22] move schema loading/validating to app config

---
 staging_service/app.py                        | 27 ++++++++++++-------
 .../individual_parsers.py                     |  6 ++---
 .../test_individual_parsers.py                |  7 ++---
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index e607641f..9ff2484f 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -6,10 +6,12 @@
 import sys
 from collections import defaultdict
 from pathlib import Path as PathPy
+from typing import Any
 from urllib.parse import parse_qs, unquote
 
 import aiohttp_cors
 from aiohttp import web
+import jsonschema
 
 from .app_error_formatter import format_import_spec_errors
 from .auth2Client import KBaseAuth2
@@ -110,21 +112,16 @@ def _file_type_resolver(path: PathPy) -> FileTypeResolution:
 
 
 def _make_dts_file_resolver() -> Callable[[Path], FileTypeResolution]:
-    """Makes a DTS file resolver.
+    """Makes a DTS file resolver
 
-    This looks a little goofy, but it ensures that the DTS manifest schema file
-    only gets loaded once per API call, no matter how many DTS manifest files are
-    expected to be parsed. It also prevents having it stick around in memory.
+    This injects the DTS schema into the FileTypeResolution's parser call.
     """
-    with open(_DTS_MANIFEST_SCHEMA) as schema_file:
-        dts_schema = json.load(schema_file)
-
     def dts_file_resolver(path: PathPy) -> FileTypeResolution:
         # must be a ".json" file
         suffix = path.suffix[1:] if path.suffix else NO_EXTENSION
         if suffix.lower() != JSON_EXTENSION:
             return FileTypeResolution(unsupported_type=suffix)
-        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, dts_schema))
+        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, _DTS_MANIFEST_SCHEMA))
 
     return dts_file_resolver
 
@@ -611,6 +608,16 @@ async def authorize_request(request):
     return username
 
 
+def load_and_validate_schema(schema_path: PathPy) -> dict[str, Any]:
+    with open(schema_path) as schema_file:
+        dts_schema = json.load(schema_file)
+    try:
+        jsonschema.Draft202012Validator.check_schema(dts_schema)
+    except jsonschema.exceptions.SchemaError as err:
+        Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}")
+    return dts_schema
+
+
 def inject_config_dependencies(config):
     """
     # TODO this is pretty hacky dependency injection
@@ -656,8 +663,10 @@ def inject_config_dependencies(config):
 
     if Path._DTS_MANIFEST_SCHEMA_PATH is None:
         raise Exception("Please provide DTS_MANIFEST_SCHEMA in the config file")
+
     global _DTS_MANIFEST_SCHEMA
-    _DTS_MANIFEST_SCHEMA = DTS_MANIFEST_SCHEMA_PATH
+    # will raise an Exception if the schema is invalid
+    _DTS_MANIFEST_SCHEMA = load_and_validate_schema(DTS_MANIFEST_SCHEMA_PATH)
 
     if FILE_EXTENSION_MAPPINGS is None:
         raise Exception("Please provide FILE_EXTENSION_MAPPINGS in the config file ")
diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index f5ca0b64..4573afe8 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -344,6 +344,9 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
     and its value will be a Tuple of frozendicts of the parameters. Also, in keeping
     with the xsv parsers, each parameter value is expected to be a PRIMITIVE_TYPE.
 
+    Note that the dts_manifest_schema is expected to be valid, and may throw an
+    unexpected exception otherwise.
+
     TODO: include further details here, and in separate documentation - ADR?
     """
     spcsrc = SpecificationSource(path)
@@ -351,7 +354,6 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
     # dummy for now
     results = {}
     try:
-        jsonschema.Draft202012Validator.check_schema(dts_manifest_schema)
         with open(path, "r") as manifest:
             manifest_json = json.load(manifest)
         if not isinstance(manifest_json, dict):
@@ -363,8 +365,6 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
             if err_path:
                 err_str += f" at {err_path}"
             errors.append(Error(ErrorType.PARSE_FAIL, err_str, spcsrc))
-    except jsonschema.exceptions.SchemaError:
-        return _error(Error(ErrorType.OTHER, "Manifest schema is invalid", spcsrc))
     except json.JSONDecodeError:
         return _error(Error(ErrorType.PARSE_FAIL, "File must be in JSON format", spcsrc))
     except FileNotFoundError:
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 87a63d41..3d9e4aeb 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -879,11 +879,8 @@ def test_dts_manifest_file_is_directory(
 @pytest.mark.parametrize("bad_schema", [None, 1, [], {"foo"}])
 def test_dts_manifest_bad_schema(bad_schema):
     f = _get_test_file("manifest_small.json")
-    _dts_manifest_parse_fail(
-        f,
-        bad_schema,
-        [Error(ErrorType.OTHER, "Manifest schema is invalid", SpecificationSource(f))],
-    )
+    with pytest.raises(Exception):
+        parse_dts_manifest(f, bad_schema)
 
 
 def test_dts_manifest_no_top_level_keys(

From 5469ce7643b39911a3dfc0d5e210e41c691cd043 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 17:16:04 -0500
Subject: [PATCH 14/22] add empty tests for schema

---
 staging_service/app.py |  2 +-
 tests/test_app.py      | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index 9ff2484f..adf255aa 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -614,7 +614,7 @@ def load_and_validate_schema(schema_path: PathPy) -> dict[str, Any]:
     try:
         jsonschema.Draft202012Validator.check_schema(dts_schema)
     except jsonschema.exceptions.SchemaError as err:
-        Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}")
+        raise Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}")
     return dts_schema
 
 
diff --git a/tests/test_app.py b/tests/test_app.py
index f8147154..1f46171b 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -1253,6 +1253,17 @@ async def test_bulk_specification_dts_fail_wrong_extension(manifest: str, expect
             assert resp.status == 400
 
 
+def test_bulk_specification_dts_fail_bad_schema():
+    # TODO: This is tested manually, as there's no good way to inject bad configs
+    # to individual tests right now.
+    # TODO: automated tests for:
+    # * missing schema config
+    # * missing schema file
+    # * malformed schema file (i.e. not json)
+    # * bad schema (good JSON, invalid as json schema)
+    pass
+
+
 async def test_bulk_specification_fail_no_files():
     async with AppClient(config) as cli:
         for f in ["", "?files=", "?files=  ,   ,,   ,  "]:

From a10dc0351af393ba4dab41ad63bc2b615caaa975 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 17:17:07 -0500
Subject: [PATCH 15/22] rename schema file

---
 .../schema/{dts_manifest.json => dts_manifest_schema.json}        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename import_specifications/schema/{dts_manifest.json => dts_manifest_schema.json} (100%)

diff --git a/import_specifications/schema/dts_manifest.json b/import_specifications/schema/dts_manifest_schema.json
similarity index 100%
rename from import_specifications/schema/dts_manifest.json
rename to import_specifications/schema/dts_manifest_schema.json

From b010cded4a3ebb47047abdf20ceaac91744193d6 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 17:19:26 -0500
Subject: [PATCH 16/22] ruff formatting

---
 staging_service/app.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/staging_service/app.py b/staging_service/app.py
index adf255aa..e5b10a9e 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -116,6 +116,7 @@ def _make_dts_file_resolver() -> Callable[[Path], FileTypeResolution]:
 
     This injects the DTS schema into the FileTypeResolution's parser call.
     """
+
     def dts_file_resolver(path: PathPy) -> FileTypeResolution:
         # must be a ".json" file
         suffix = path.suffix[1:] if path.suffix else NO_EXTENSION

From af30bf28202fd8a3571b3a4fba65ee0cb2f0ce39 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 17:21:53 -0500
Subject: [PATCH 17/22] fix configs

---
 deployment/conf/deployment.cfg | 2 +-
 deployment/conf/testing.cfg    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/conf/deployment.cfg b/deployment/conf/deployment.cfg
index 5ec9ac74..b6622c58 100644
--- a/deployment/conf/deployment.cfg
+++ b/deployment/conf/deployment.cfg
@@ -4,4 +4,4 @@ DATA_DIR = /kb/deployment/lib/src/data/bulk/
 AUTH_URL = https://ci.kbase.us/services/auth/api/V2/token
 CONCIERGE_PATH = /kbaseconcierge
 FILE_EXTENSION_MAPPINGS = /kb/deployment/conf/supported_apps_w_extensions.json
-DTS_MANIFEST_SCHEMA = /kb/deployment/import_specifications/schema/dts_manifest.json
+DTS_MANIFEST_SCHEMA = /kb/deployment/import_specifications/schema/dts_manifest_schema.json
diff --git a/deployment/conf/testing.cfg b/deployment/conf/testing.cfg
index 41f067bb..5e016215 100644
--- a/deployment/conf/testing.cfg
+++ b/deployment/conf/testing.cfg
@@ -5,4 +5,4 @@ AUTH_URL = https://ci.kbase.us/services/auth/api/V2/token
 CONCIERGE_PATH = /kbaseconcierge
 FILE_EXTENSION_MAPPINGS = ./deployment/conf/supported_apps_w_extensions.json
 ;FILE_EXTENSION_MAPPINGS_PYCHARM = ../deployment/conf/supported_apps_w_extensions.json
-DTS_MANIFEST_SCHEMA = ./import_specifications/schema/dts_manifest.json
+DTS_MANIFEST_SCHEMA = ./import_specifications/schema/dts_manifest_schema.json

From ebf25e74d0a5a4803589589f7f21a7aa2c3f26af Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 21:58:23 -0500
Subject: [PATCH 18/22] updates to config, validator, etc.

---
 staging_service/app.py                        | 19 ++++----
 .../individual_parsers.py                     | 11 ++---
 .../test_individual_parsers.py                | 46 ++++++++-----------
 tests/test_app.py                             | 41 +++++++++++++++++
 4 files changed, 73 insertions(+), 44 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index e5b10a9e..14922cd6 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -6,7 +6,6 @@
 import sys
 from collections import defaultdict
 from pathlib import Path as PathPy
-from typing import Any
 from urllib.parse import parse_qs, unquote
 
 import aiohttp_cors
@@ -44,7 +43,7 @@
 VERSION = "1.3.6"
 
 _DATATYPE_MAPPINGS = None
-_DTS_MANIFEST_SCHEMA = None
+_DTS_MANIFEST_VALIDATOR: jsonschema.Draft202012Validator | None = None
 
 _APP_JSON = "application/json"
 
@@ -122,7 +121,7 @@ def dts_file_resolver(path: PathPy) -> FileTypeResolution:
         suffix = path.suffix[1:] if path.suffix else NO_EXTENSION
         if suffix.lower() != JSON_EXTENSION:
             return FileTypeResolution(unsupported_type=suffix)
-        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, _DTS_MANIFEST_SCHEMA))
+        return FileTypeResolution(parser=lambda p: parse_dts_manifest(p, _DTS_MANIFEST_VALIDATOR))
 
     return dts_file_resolver
 
@@ -609,14 +608,14 @@ async def authorize_request(request):
     return username
 
 
-def load_and_validate_schema(schema_path: PathPy) -> dict[str, Any]:
+def load_and_validate_schema(schema_path: PathPy) -> jsonschema.Draft202012Validator:
     with open(schema_path) as schema_file:
         dts_schema = json.load(schema_file)
     try:
         jsonschema.Draft202012Validator.check_schema(dts_schema)
     except jsonschema.exceptions.SchemaError as err:
-        raise Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}")
-    return dts_schema
+        raise Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}") from err
+    return jsonschema.Draft202012Validator(dts_schema)
 
 
 def inject_config_dependencies(config):
@@ -651,7 +650,7 @@ def inject_config_dependencies(config):
     Path._DATA_DIR = DATA_DIR
     Path._META_DIR = META_DIR
     Path._CONCIERGE_PATH = CONCIERGE_PATH
-    Path._DTS_MANIFEST_SCHEMA_PATH = DTS_MANIFEST_SCHEMA_PATH
+    _DTS_MANIFEST_SCHEMA_PATH = DTS_MANIFEST_SCHEMA_PATH
 
     if Path._DATA_DIR is None:
         raise Exception("Please provide DATA_DIR in the config file ")
@@ -662,12 +661,12 @@ def inject_config_dependencies(config):
     if Path._CONCIERGE_PATH is None:
         raise Exception("Please provide CONCIERGE_PATH in the config file ")
 
-    if Path._DTS_MANIFEST_SCHEMA_PATH is None:
+    if _DTS_MANIFEST_SCHEMA_PATH is None:
         raise Exception("Please provide DTS_MANIFEST_SCHEMA in the config file")
 
-    global _DTS_MANIFEST_SCHEMA
+    global _DTS_MANIFEST_VALIDATOR
     # will raise an Exception if the schema is invalid
-    _DTS_MANIFEST_SCHEMA = load_and_validate_schema(DTS_MANIFEST_SCHEMA_PATH)
+    _DTS_MANIFEST_VALIDATOR = load_and_validate_schema(DTS_MANIFEST_SCHEMA_PATH)
 
     if FILE_EXTENSION_MAPPINGS is None:
         raise Exception("Please provide FILE_EXTENSION_MAPPINGS in the config file ")
diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py
index 4573afe8..e0379caf 100644
--- a/staging_service/import_specifications/individual_parsers.py
+++ b/staging_service/import_specifications/individual_parsers.py
@@ -4,13 +4,12 @@
 
 import csv
 import json
-import jsonschema
 import math
 import re
 from pathlib import Path
 from typing import Any, Optional, Tuple, Union
 
-import jsonschema.exceptions
+from jsonschema import Draft202012Validator
 import magic
 import pandas
 from frozendict import frozendict
@@ -321,7 +320,7 @@ def parse_excel(path: Path) -> ParseResults:
         return _error(Error(ErrorType.PARSE_FAIL, "No non-header data in file", spcsrc))
 
 
-def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
+def parse_dts_manifest(path: Path, validator: Draft202012Validator) -> ParseResults:
     """
     Parse the provided DTS manifest file. Expected to be JSON, and will fail otherwise.
     The manifest should have this format, with expected keys included:
@@ -344,10 +343,7 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
     and its value will be a Tuple of frozendicts of the parameters. Also, in keeping
     with the xsv parsers, each parameter value is expected to be a PRIMITIVE_TYPE.
 
-    Note that the dts_manifest_schema is expected to be valid, and may throw an
-    unexpected exception otherwise.
-
-    TODO: include further details here, and in separate documentation - ADR?
+    TODO: include further details in separate documentation
     """
     spcsrc = SpecificationSource(path)
     errors = []
@@ -358,7 +354,6 @@ def parse_dts_manifest(path: Path, dts_manifest_schema: dict) -> ParseResults:
             manifest_json = json.load(manifest)
         if not isinstance(manifest_json, dict):
             return _error(Error(ErrorType.PARSE_FAIL, "Manifest is not a dictionary", spcsrc))
-        validator = jsonschema.Draft202012Validator(dts_manifest_schema)
         for err in validator.iter_errors(manifest_json):
             err_str = err.message
             err_path = list(err.absolute_path)
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index 3d9e4aeb..a719e5ea 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -4,7 +4,6 @@
 import uuid
 from collections.abc import Callable, Generator
 from pathlib import Path
-from typing import Any
 
 # TODO update to C impl when fixed: https://github.com/Marco-Sulla/python-frozendict/issues/26
 from frozendict import frozendict
@@ -21,6 +20,8 @@
 )
 from tests.test_app import FileUtil
 from tests.test_utils import bootstrap_config
+from jsonschema import Draft202012Validator
+
 
 _TEST_DATA_DIR = (Path(__file__).parent / "test_data").resolve()
 
@@ -36,11 +37,11 @@ def temp_dir_fixture() -> Generator[Path, None, None]:
 
 
 @pytest.fixture(scope="module")
-def dts_schema() -> Generator[dict[str, Any], None, None]:
+def dts_validator() -> Generator[Draft202012Validator, None, None]:
     config = bootstrap_config()
     with open(config["staging_service"]["DTS_MANIFEST_SCHEMA"]) as dts_schema_file:
         schema = json.load(dts_schema_file)
-        yield schema
+        yield Draft202012Validator(schema)
 
 
 ##########################################
@@ -778,9 +779,9 @@ def test_excel_parse_fail_unequal_rows():
     )
 
 
-def test_dts_manifest_parse_success(dts_schema: dict[str, Any]):
+def test_dts_manifest_parse_success(dts_validator: Draft202012Validator):
     f = _get_test_file("manifest_small.json")
-    res = parse_dts_manifest(f, dts_schema)
+    res = parse_dts_manifest(f, dts_validator)
     # fails for now
     assert res.results is None
     assert res.errors == tuple(
@@ -805,25 +806,25 @@ def manifest_writer(input_json: dict | list) -> Path:
     return manifest_writer
 
 
-def _dts_manifest_parse_fail(input_file: Path, schema: dict, errors: list[Error]):
+def _dts_manifest_parse_fail(input_file: Path, validator: Draft202012Validator, errors: list[Error]):
     """
     Tests a failing DTS manifest parse.
     input_file - the path to the input file. Might be a directory or not exist.
     errors - a list of Error objects expected to be in the order generated by
         the call to parse_dts_manifest.
     """
-    res = parse_dts_manifest(input_file, schema)
+    res = parse_dts_manifest(input_file, validator)
     assert res.results is None
     assert res.errors == tuple(errors)
 
 
-def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
+def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator):
     test_file = temp_dir / str(uuid.uuid4())
     with open(test_file, "w", encoding="utf-8") as outfile:
         outfile.write("totally not json")
     _dts_manifest_parse_fail(
         test_file,
-        dts_schema,
+        dts_validator,
         [
             Error(
                 ErrorType.PARSE_FAIL, "File must be in JSON format", SpecificationSource(test_file)
@@ -833,12 +834,12 @@ def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_schema
 
 
 def test_dts_manifest_non_dict(
-    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+    write_dts_manifest: Callable[[dict | list], Path], dts_validator: Draft202012Validator
 ):
     manifest_path = write_dts_manifest(["wrong_format"])
     _dts_manifest_parse_fail(
         manifest_path,
-        dts_schema,
+        dts_validator,
         [
             Error(
                 ErrorType.PARSE_FAIL,
@@ -849,23 +850,23 @@ def test_dts_manifest_non_dict(
     )
 
 
-def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]):
+def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator):
     manifest_path = temp_dir / "not_a_file"
     _dts_manifest_parse_fail(
         manifest_path,
-        dts_schema,
+        dts_validator,
         [Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(manifest_path))],
     )
 
 
 def test_dts_manifest_file_is_directory(
-    temp_dir: Generator[Path, None, None], dts_schema: dict[str, Any]
+    temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator
 ):
     test_file = temp_dir / "testdir.json"
     os.makedirs(test_file, exist_ok=True)
     _dts_manifest_parse_fail(
         test_file,
-        dts_schema,
+        dts_validator,
         [
             Error(
                 ErrorType.PARSE_FAIL,
@@ -876,20 +877,13 @@ def test_dts_manifest_file_is_directory(
     )
 
 
-@pytest.mark.parametrize("bad_schema", [None, 1, [], {"foo"}])
-def test_dts_manifest_bad_schema(bad_schema):
-    f = _get_test_file("manifest_small.json")
-    with pytest.raises(Exception):
-        parse_dts_manifest(f, bad_schema)
-
-
 def test_dts_manifest_no_top_level_keys(
-    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+    write_dts_manifest: Callable[[dict | list], Path], dts_validator: Draft202012Validator
 ):
     manifest_path = write_dts_manifest({"missing": "stuff"})
     _dts_manifest_parse_fail(
         manifest_path,
-        dts_schema,
+        dts_validator,
         [
             Error(
                 ErrorType.PARSE_FAIL,
@@ -906,7 +900,7 @@ def test_dts_manifest_no_top_level_keys(
 
 
 def test_dts_manifest_fail_with_path(
-    write_dts_manifest: Callable[[dict | list], Path], dts_schema: dict[str, Any]
+    write_dts_manifest: Callable[[dict | list], Path], dts_validator: Draft202012Validator
 ):
     manifest_path = write_dts_manifest(
         {
@@ -919,7 +913,7 @@ def test_dts_manifest_fail_with_path(
     )
     _dts_manifest_parse_fail(
         manifest_path,
-        dts_schema,
+        dts_validator,
         [
             Error(
                 ErrorType.PARSE_FAIL,
diff --git a/tests/test_app.py b/tests/test_app.py
index 1f46171b..899c1c6a 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -4,6 +4,8 @@
 import json
 import os
 import platform
+import uuid
+import jsonschema
 import pytest
 import shutil
 import string
@@ -1264,6 +1266,45 @@ def test_bulk_specification_dts_fail_bad_schema():
     pass
 
 
+def test_load_and_validate_schema_good():
+    # TODO: update this after updating how config is handled
+    schema_file = config["staging_service"]["DTS_MANIFEST_SCHEMA"]
+    validator = app.load_and_validate_schema(schema_file)
+    assert isinstance(validator, jsonschema.Draft202012Validator)
+
+
+def test_load_and_validate_schema_missing_file():
+    not_real_file = Path("not_real")
+    while not_real_file.exists():
+        not_real_file = Path(str(uuid.uuid4()))
+    with pytest.raises(FileNotFoundError, match="No such file or directory"):
+        app.load_and_validate_schema(not_real_file)
+
+
+def test_load_and_validate_schema_malformed_file(tmp_path: Path):
+    # TODO: migrate FileUtil and import_specifications.test_individual_parsers.temp_path_fixture
+    # into conftest.py, and resolve everywhere else that FileUtil gets used.
+    # Until then, the built-in tmp_path is appropriate for these tests
+    wrong_schema = "not valid json"
+    schema_file = tmp_path / f"{uuid.uuid4()}.json"
+    schema_file.write_text(wrong_schema, encoding="utf-8")
+    with pytest.raises(json.JSONDecodeError, match="Expecting value: line 1 column 1"):
+        app.load_and_validate_schema(schema_file)
+
+
+def test_load_and_validate_schema_bad(tmp_path: Path):
+    invalid = {
+        "properties": {
+            "some_prop": { "type": "not_real"}
+        }
+    }
+    schema_file = tmp_path / f"{uuid.uuid4()}.json"
+    schema_file.write_text(json.dumps(invalid), encoding="utf-8")
+    exp_err = f"Schema file {schema_file} is not a valid JSON schema: 'not_real' is not valid"
+    with pytest.raises(Exception, match=exp_err):
+        app.load_and_validate_schema(schema_file)
+
+
 async def test_bulk_specification_fail_no_files():
     async with AppClient(config) as cli:
         for f in ["", "?files=", "?files=  ,   ,,   ,  "]:

From cb4b95a0287c36adb1c9827acffc7ab3025469db Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Wed, 11 Dec 2024 21:58:47 -0500
Subject: [PATCH 19/22] ruff formatting

---
 staging_service/app.py                               |  4 +++-
 .../import_specifications/test_individual_parsers.py | 12 +++++++++---
 tests/test_app.py                                    |  6 +-----
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index 14922cd6..4c34f841 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -614,7 +614,9 @@ def load_and_validate_schema(schema_path: PathPy) -> jsonschema.Draft202012Valid
     try:
         jsonschema.Draft202012Validator.check_schema(dts_schema)
     except jsonschema.exceptions.SchemaError as err:
-        raise Exception(f"Schema file {schema_path} is not a valid JSON schema: {err.message}") from err
+        raise Exception(
+            f"Schema file {schema_path} is not a valid JSON schema: {err.message}"
+        ) from err
     return jsonschema.Draft202012Validator(dts_schema)
 
 
diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py
index a719e5ea..8c6c3af4 100644
--- a/tests/import_specifications/test_individual_parsers.py
+++ b/tests/import_specifications/test_individual_parsers.py
@@ -806,7 +806,9 @@ def manifest_writer(input_json: dict | list) -> Path:
     return manifest_writer
 
 
-def _dts_manifest_parse_fail(input_file: Path, validator: Draft202012Validator, errors: list[Error]):
+def _dts_manifest_parse_fail(
+    input_file: Path, validator: Draft202012Validator, errors: list[Error]
+):
     """
     Tests a failing DTS manifest parse.
     input_file - the path to the input file. Might be a directory or not exist.
@@ -818,7 +820,9 @@ def _dts_manifest_parse_fail(input_file: Path, validator: Draft202012Validator,
     assert res.errors == tuple(errors)
 
 
-def test_dts_manifest_non_json(temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator):
+def test_dts_manifest_non_json(
+    temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator
+):
     test_file = temp_dir / str(uuid.uuid4())
     with open(test_file, "w", encoding="utf-8") as outfile:
         outfile.write("totally not json")
@@ -850,7 +854,9 @@ def test_dts_manifest_non_dict(
     )
 
 
-def test_dts_manifest_not_found(temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator):
+def test_dts_manifest_not_found(
+    temp_dir: Generator[Path, None, None], dts_validator: Draft202012Validator
+):
     manifest_path = temp_dir / "not_a_file"
     _dts_manifest_parse_fail(
         manifest_path,
diff --git a/tests/test_app.py b/tests/test_app.py
index 899c1c6a..07d167db 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -1293,11 +1293,7 @@ def test_load_and_validate_schema_malformed_file(tmp_path: Path):
 
 
 def test_load_and_validate_schema_bad(tmp_path: Path):
-    invalid = {
-        "properties": {
-            "some_prop": { "type": "not_real"}
-        }
-    }
+    invalid = {"properties": {"some_prop": {"type": "not_real"}}}
     schema_file = tmp_path / f"{uuid.uuid4()}.json"
     schema_file.write_text(json.dumps(invalid), encoding="utf-8")
     exp_err = f"Schema file {schema_file} is not a valid JSON schema: 'not_real' is not valid"

From cf6c9f326d4dd0c44a8277a32e554a810e99c7b4 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Thu, 12 Dec 2024 20:41:43 -0500
Subject: [PATCH 20/22] removing unused variable

---
 staging_service/app.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/staging_service/app.py b/staging_service/app.py
index 4c34f841..7d736363 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -652,7 +652,6 @@ def inject_config_dependencies(config):
     Path._DATA_DIR = DATA_DIR
     Path._META_DIR = META_DIR
     Path._CONCIERGE_PATH = CONCIERGE_PATH
-    _DTS_MANIFEST_SCHEMA_PATH = DTS_MANIFEST_SCHEMA_PATH
 
     if Path._DATA_DIR is None:
         raise Exception("Please provide DATA_DIR in the config file ")
@@ -663,7 +662,7 @@ def inject_config_dependencies(config):
     if Path._CONCIERGE_PATH is None:
         raise Exception("Please provide CONCIERGE_PATH in the config file ")
 
-    if _DTS_MANIFEST_SCHEMA_PATH is None:
+    if DTS_MANIFEST_SCHEMA_PATH is None:
         raise Exception("Please provide DTS_MANIFEST_SCHEMA in the config file")
 
     global _DTS_MANIFEST_VALIDATOR

From 711808b23bceae41480d1296553ca9c615fee4d9 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Thu, 12 Dec 2024 20:56:00 -0500
Subject: [PATCH 21/22] start fake files with uuid

---
 tests/test_app.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_app.py b/tests/test_app.py
index 07d167db..1f78e849 100644
--- a/tests/test_app.py
+++ b/tests/test_app.py
@@ -1274,7 +1274,9 @@ def test_load_and_validate_schema_good():
 
 
 def test_load_and_validate_schema_missing_file():
-    not_real_file = Path("not_real")
+    not_real_file = Path(str(uuid.uuid4()))
+    # double-check that the file doesn't exist and make a new one if so.
+    # shouldn't ever happen, ideally, but easy to check.
     while not_real_file.exists():
         not_real_file = Path(str(uuid.uuid4()))
     with pytest.raises(FileNotFoundError, match="No such file or directory"):

From 05b619b1ecbccf59e2986278cdc6f91ade38ca85 Mon Sep 17 00:00:00 2001
From: Bill Riehl <wjriehl@lbl.gov>
Date: Thu, 12 Dec 2024 21:03:04 -0500
Subject: [PATCH 22/22] add TODO

---
 staging_service/app.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/staging_service/app.py b/staging_service/app.py
index 7d736363..7c329b50 100644
--- a/staging_service/app.py
+++ b/staging_service/app.py
@@ -609,6 +609,14 @@ async def authorize_request(request):
 
 
 def load_and_validate_schema(schema_path: PathPy) -> jsonschema.Draft202012Validator:
+    """Loads and validates a JSON schema from a path.
+
+    This expects a JSON schema loaded that validates under the 2020-12 draft schema
+    format: https://json-schema.org/draft/2020-12
+
+    This is tested directly as a function in test_app.py, but the whole workflow when
+    the app server is run is only tested manually.
+    """
     with open(schema_path) as schema_file:
         dts_schema = json.load(schema_file)
     try:
@@ -667,6 +675,8 @@ def inject_config_dependencies(config):
 
     global _DTS_MANIFEST_VALIDATOR
     # will raise an Exception if the schema is invalid
+    # TODO: write automated tests that exercise this code under different config
+    # conditions and error states.
     _DTS_MANIFEST_VALIDATOR = load_and_validate_schema(DTS_MANIFEST_SCHEMA_PATH)
 
     if FILE_EXTENSION_MAPPINGS is None: