Merge pull request #554 from IATI/new_advisory_checks

New advisory check to see if cross-referenced IATI Identifiers exist
IATI · Apr 3, 2024 · 0a5498f · 0a5498f
2 parents 7582cab + 4286085
commit 0a5498f
Show file tree

Hide file tree

Showing 32 changed files with 1,683,013 additions and 172 deletions.
diff --git a/.env.example b/.env.example
@@ -9,4 +9,9 @@ REDIS_HOSTNAME=redis  # replace with localhost if you run outside docker
 
 VALIDATOR_SERVICES_URL=https://dev-func-validator-services.azurewebsites.net/api
 VALIDATOR_SERVICES_KEY_NAME=x-functions-key
-VALIDATOR_SERVICES_KEY_VALUE=
+VALIDATOR_SERVICES_KEY_VALUE=
+
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000
diff --git a/.github/workflows/develop-func-deploy.yml b/.github/workflows/develop-func-deploy.yml
@@ -33,6 +33,10 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.DEV_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.DEV_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.DEV_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   should_run:
@@ -138,6 +142,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 

diff --git a/.github/workflows/prod-func-deploy.yml b/.github/workflows/prod-func-deploy.yml
@@ -23,6 +23,10 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.PROD_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.PROD_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.PROD_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   build-test-deploy:
@@ -123,6 +127,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 

diff --git a/README.md b/README.md
@@ -99,6 +99,14 @@ VALIDATOR_SERVICES_KEY_VALUE=
 
 -   URL and API Key for Validator Services, used to get list of Publisher Identifiers
 
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000
+
+- URL and API Key for datastore services, used by the advisory system to check for the
+  existence of IATI Identifiers in the Datastore
+
 ### App config defaults (set in `config/config.js`)
 
 ```

diff --git a/advisory-definitions/advisoryDefinitions-2.01.json b/advisory-definitions/advisoryDefinitions-2.01.json
@@ -0,0 +1,32 @@
+[   
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/advisory-definitions/advisoryDefinitions-2.02.json b/advisory-definitions/advisoryDefinitions-2.02.json
@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/advisory-definitions/advisoryDefinitions-2.03.json b/advisory-definitions/advisoryDefinitions-2.03.json
@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/config/config.js b/config/config.js
@@ -19,6 +19,10 @@ const config = {
     VALIDATOR_SERVICES_URL: process.env.VALIDATOR_SERVICES_URL,
     VALIDATOR_SERVICES_KEY_NAME: process.env.VALIDATOR_SERVICES_KEY_NAME,
     VALIDATOR_SERVICES_KEY_VALUE: process.env.VALIDATOR_SERVICES_KEY_VALUE,
+    DATASTORE_SERVICES_URL: process.env.DATASTORE_SERVICES_URL,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE,
+    DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: process.env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS
 };
 
 export default config;