Merge pull request #1312 from Sage-Bionetworks/develop-api-validation…

…-asset-view-FDS-810

Fix an issue where the asset view was not being set correctly for operations involving manifest validation

schematic_api/api/openapi/api.yaml

@@ -82,7 +82,7 @@ paths:
           schema:
             type: string
             nullable: true
-          description: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           required: false
         - in: query
           name: output_format
@@ -167,7 +167,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn28559058
           required: true
         - in: query
@@ -261,6 +261,24 @@ paths:
           "Cancer Type": "Breast",
           "Family History": "Breast, Lung",
           }]'
+        - in: query
+          name: asset_view
+          schema:
+            type: string
+            nullable: false
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project (i.e. master_fileview_id in config_example.yml). Required for validating manifests involving cross-manifest validation.
+          example: syn23643253
+          required: false
+        - in: query
+          name: project_scope
+          schema:
+            type: array
+            items: 
+              type: string
+            nullable: false
+          description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse. Relevant for validating manifests involving cross-manifest validation, but optional.
+          example: ['syn23643250', 'syn47218127', 'syn47218347']
+          required: false
 
       operationId: schematic_api.api.routes.validate_manifest_route
       responses:
@@ -346,8 +364,8 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
-          example: syn28559058
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
+          example: syn23643253
           required: true
         - in: query
           name: json_str
@@ -379,6 +397,16 @@ paths:
             type: boolean
             default: true
           required: false
+        - in: query
+          name: project_scope
+          schema:
+            type: array
+            items: 
+              type: string
+            nullable: false
+          description: List, a subset of the projects contained within the asset view that are relevant for the current operation. Speeds up some operations that interact with Synapse.
+          example: ['syn23643250', 'syn47218127', 'syn47218347']
+          required: false
       operationId: schematic_api.api.routes.submit_manifest_route
       responses:
         "200":
@@ -498,7 +526,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
         - in: query
@@ -529,7 +557,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
       responses:
@@ -552,7 +580,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
         - in: query
@@ -583,7 +611,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
         - in: query
@@ -630,7 +658,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
         - in: query
@@ -669,7 +697,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
       responses:
@@ -727,7 +755,7 @@ paths:
           schema:
             type: string
             nullable: false
-          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview_id in config_example.yml)
           example: syn23643253
           required: true
         - in: query

schematic_api/api/routes.py

@@ -339,7 +339,7 @@ def create_single_manifest(data_type, title, dataset_id=None, output_format=None
 
 #####profile validate manifest route function 
 #@profile(sort_by='cumulative', strip_dirs=True)
-def validate_manifest_route(schema_url, data_type, restrict_rules=None, json_str=None):
+def validate_manifest_route(schema_url, data_type, restrict_rules=None, json_str=None, asset_view=None, project_scope = None):
     # Access token now stored in request header
     access_token = get_access_token()
 
@@ -348,7 +348,7 @@ def validate_manifest_route(schema_url, data_type, restrict_rules=None, json_str
         restrict_rules=False
 
     # call config_handler()
-    config_handler()
+    config_handler(asset_view = asset_view)
 
     #If restrict_rules parameter is set to None, then default it to False 
     if not restrict_rules:
@@ -370,7 +370,7 @@ def validate_manifest_route(schema_url, data_type, restrict_rules=None, json_str
     )
 
     errors, warnings = metadata_model.validateModelManifest(
-        manifestPath=temp_path, rootNode=data_type, restrict_rules=restrict_rules, access_token=access_token
+        manifestPath=temp_path, rootNode=data_type, restrict_rules=restrict_rules, project_scope=project_scope, access_token=access_token
     )
 
     res_dict = {"errors": errors, "warnings": warnings}
@@ -379,7 +379,15 @@ def validate_manifest_route(schema_url, data_type, restrict_rules=None, json_str
 
 #####profile validate manifest route function 
 #@profile(sort_by='cumulative', strip_dirs=True)
-def submit_manifest_route(schema_url, asset_view=None, manifest_record_type=None, json_str=None, table_manipulation=None, data_type=None, hide_blanks=False):
+def submit_manifest_route(schema_url, 
+                          asset_view=None, 
+                          manifest_record_type=None, 
+                          json_str=None, 
+                          table_manipulation=None, 
+                          data_type=None, 
+                          hide_blanks=False, 
+                          project_scope=None,
+                          ):
     # call config_handler()
     config_handler(asset_view = asset_view)
 
@@ -427,7 +435,9 @@ def submit_manifest_route(schema_url, asset_view=None, manifest_record_type=None
         restrict_rules = restrict_rules, 
         hide_blanks=hide_blanks,
         table_manipulation = table_manipulation, 
-        use_schema_label=use_schema_label)
+        use_schema_label=use_schema_label,
+        project_scope=project_scope,
+        )
 
     return manifest_id
 

tests/conftest.py

@@ -63,7 +63,7 @@ def get_schema_explorer(path=None, *paths):
         return se
 
     @staticmethod
-    def get_python_version(self):
+    def get_python_version():
         version=sys.version
         base_version=".".join(version.split('.')[0:2])
 
@@ -72,7 +72,7 @@ def get_python_version(self):
     @staticmethod
     def get_python_project(self):
 
-        version = self.get_python_version(Helpers)
+        version = self.get_python_version()
 
         python_projects = {
             "3.7":  "syn47217926",

tests/test_api.py

@@ -773,21 +773,44 @@ def test_submit_manifest_table_and_file_replace(self, client, request_headers, d
 
     @pytest.mark.synapse_credentials_needed
     @pytest.mark.submission
-    def test_submit_manifest_file_only_replace(self, client, request_headers, data_model_jsonld, test_manifest_submit):
+    @pytest.mark.parametrize("data_type, manifest_path_fixture",[("Biospecimen","test_manifest_submit"), ("MockComponent", "test_manifest_csv")])
+    def test_submit_manifest_file_only_replace(self, helpers, client, request_headers, data_model_jsonld, data_type, manifest_path_fixture, request):
         """Testing submit manifest in a csv format as a file
         """
         params = {
             "schema_url": data_model_jsonld,
-            "data_type": "Biospecimen",
+            "data_type": data_type,
             "restrict_rules": False, 
             "manifest_record_type": "file_only",
-            "asset_view": "syn51514344",
-            "dataset_id": "syn51514345",
             "table_manipulation": 'replace',
             "use_schema_label": True
         }
-        response_csv = client.post('http://localhost:3001/v1/model/submit', query_string=params, data={"file_name": (open(test_manifest_submit, 'rb'), "test.csv")}, headers=request_headers)
+
+        if data_type == "Biospecimen":
+            specific_params = {
+            "asset_view": "syn51514344",
+            "dataset_id": "syn51514345",
+            }
+
+        elif data_type == "MockComponent":
+            python_version = helpers.get_python_version()
+
+            if python_version == "3.10":
+                dataset_id = "syn52656106"
+            elif python_version =="3.9":
+                dataset_id = "syn52656104"
+
+            specific_params = {
+            "asset_view": "syn23643253",
+            "dataset_id": dataset_id
+            }
+
+        params.update(specific_params)
+
+        manifest_path = request.getfixturevalue(manifest_path_fixture)
+        response_csv = client.post('http://localhost:3001/v1/model/submit', query_string=params, data={"file_name": (open(manifest_path, 'rb'), "test.csv")}, headers=request_headers)
         assert response_csv.status_code == 200 
+
     @pytest.mark.synapse_credentials_needed    
     @pytest.mark.submission
     def test_submit_manifest_json_str_replace(self, client, request_headers, data_model_jsonld):

tests/test_store.py

@@ -65,7 +65,7 @@ def dataset_fileview_table_tidy(dataset_fileview, dataset_fileview_table):
 @pytest.fixture
 def version(synapse_store, helpers):
 
-    yield helpers.get_python_version(helpers)
+    yield helpers.get_python_version()
 
 @pytest.fixture
 def projectId(synapse_store, helpers):
@@ -75,7 +75,7 @@ def projectId(synapse_store, helpers):
 @pytest.fixture
 def datasetId(synapse_store, projectId, helpers):
     dataset = Folder(
-        name = 'Table Test  Dataset ' + helpers.get_python_version(helpers),
+        name = 'Table Test  Dataset ' + helpers.get_python_version(),
         parent = projectId,
         )