Merge pull request #557 from IATI/develop

Added networked data advisory check. Node 20.
IATI · Jun 28, 2024 · ae0dc41 · ae0dc41
2 parents 76b2ce5 + e5e4101
commit ae0dc41
Show file tree

Hide file tree

Showing 37 changed files with 1,683,096 additions and 227 deletions.
diff --git a/.env.example b/.env.example
@@ -9,4 +9,9 @@ REDIS_HOSTNAME=redis  # replace with localhost if you run outside docker
 
 VALIDATOR_SERVICES_URL=https://dev-func-validator-services.azurewebsites.net/api
 VALIDATOR_SERVICES_KEY_NAME=x-functions-key
-VALIDATOR_SERVICES_KEY_VALUE=
+VALIDATOR_SERVICES_KEY_VALUE=
+
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000
diff --git a/.github/workflows/develop-func-ci.yml b/.github/workflows/develop-func-ci.yml
@@ -21,17 +21,17 @@ jobs:
           sudo apt-get install libxml2-utils
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Setup Node Environment'
-        uses: actions/setup-node@v3.8.1
+        uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
 
       - name: 'Install Dependencies with Npm'
         run: |
-          npm i -g npm@8.19.4
+          npm i -g npm@^10
           npm ci
 
       - name: 'Run unit tests'
@@ -41,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Build Docker Image'
         run: |

diff --git a/.github/workflows/develop-func-deploy.yml b/.github/workflows/develop-func-deploy.yml
@@ -33,6 +33,10 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.DEV_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.DEV_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.DEV_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   should_run:
@@ -44,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1.4.7
@@ -138,6 +142,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 
@@ -166,7 +190,7 @@ jobs:
         run: sleep 180
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Install newman'
         run: |

diff --git a/.github/workflows/prod-func-deploy.yml b/.github/workflows/prod-func-deploy.yml
@@ -23,13 +23,17 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.PROD_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.PROD_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.PROD_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   build-test-deploy:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1.4.7
@@ -123,6 +127,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 
@@ -151,7 +175,7 @@ jobs:
         run: sleep 300
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Install newman'
         run: |

diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-lts/gallium
+lts/iron
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/azure-functions/node:4.7.2.1-node16
+FROM mcr.microsoft.com/azure-functions/node:4.33.1-node20
 
 # install libxml2-utils for xmllint
 RUN \
@@ -14,6 +14,6 @@ COPY . /home/site/wwwroot
 # Install node_modules 
 WORKDIR /home/site/wwwroot
 RUN \
-    npm i -g npm@8.19.4 && \
+    npm i -g npm@^10 && \
     npm pkg delete scripts.prepare && \
     npm ci --production
diff --git a/README.md b/README.md
@@ -1,3 +1,16 @@
+Summary
+=======
+
+Product  |  Validator API Endpoints
+--- | ---
+Description | Node.JS app that provides API end points to validate IATI XML files, is used by https://validator.iatistandard.org/
+Website |  [https://developer.iatistandard.org/](https://developer.iatistandard.org/)
+Related | [IATI/validator-services](https://github.com/IATI/validator-services), [IATI/validator-web](https://github.com/IATI/validator-web)
+Documentation | [https://developer.iatistandard.org/](https://developer.iatistandard.org/)
+Technical Issues | https://github.com/IATI/js-validator-api/issues
+Support | https://iatistandard.org/en/guidance/get-support/
+
+
 # IATI JavaScript Validator API
 
 [![Deploy_To_Dev_Function_On_Push](https://github.com/IATI/js-validator-api/actions/workflows/develop-func-deploy.yml/badge.svg)](https://github.com/IATI/js-validator-api/actions/workflows/develop-func-deploy.yml)
@@ -48,7 +61,7 @@ This should return a version number (change port if running directly instead of
 
 To test the validation is working, obtain an IATI XML file, and run the following command (this is if running via the docker setup; change port to `7071` if running directly):
 
-`curl -X POST -d @PATH_TO_IATI_XML http://localhost:8080/api/pub/validate`
+`curl -X POST --data-binary @PATH_TO_IATI_XML http://localhost:8080/api/pub/validate`
 
 You should see something like (if the file is a valid IATI file):
 
@@ -99,6 +112,14 @@ VALIDATOR_SERVICES_KEY_VALUE=
 
 -   URL and API Key for Validator Services, used to get list of Publisher Identifiers
 
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000
+
+- URL and API Key for datastore services, used by the advisory system to check for the
+  existence of IATI Identifiers in the Datastore
+
 ### App config defaults (set in `config/config.js`)
 
 ```

diff --git a/advisory-definitions/advisoryDefinitions-2.01.json b/advisory-definitions/advisoryDefinitions-2.01.json
@@ -0,0 +1,32 @@
+[   
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/advisory-definitions/advisoryDefinitions-2.02.json b/advisory-definitions/advisoryDefinitions-2.02.json
@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/advisory-definitions/advisoryDefinitions-2.03.json b/advisory-definitions/advisoryDefinitions-2.03.json
@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]
diff --git a/config/config.js b/config/config.js
@@ -19,6 +19,10 @@ const config = {
     VALIDATOR_SERVICES_URL: process.env.VALIDATOR_SERVICES_URL,
     VALIDATOR_SERVICES_KEY_NAME: process.env.VALIDATOR_SERVICES_KEY_NAME,
     VALIDATOR_SERVICES_KEY_VALUE: process.env.VALIDATOR_SERVICES_KEY_VALUE,
+    DATASTORE_SERVICES_URL: process.env.DATASTORE_SERVICES_URL,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE,
+    DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: process.env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS
 };
 
 export default config;