Added networked data advisory check. Node 20.

.env.example

@@ -9,4 +9,9 @@ REDIS_HOSTNAME=redis  # replace with localhost if you run outside docker
 
 VALIDATOR_SERVICES_URL=https://dev-func-validator-services.azurewebsites.net/api
 VALIDATOR_SERVICES_KEY_NAME=x-functions-key
-VALIDATOR_SERVICES_KEY_VALUE=
+VALIDATOR_SERVICES_KEY_VALUE=
+
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000

.github/workflows/develop-func-ci.yml

@@ -21,17 +21,17 @@ jobs:
           sudo apt-get install libxml2-utils
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Setup Node Environment'
-        uses: actions/setup-node@v3.8.1
+        uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
           cache: 'npm'
 
       - name: 'Install Dependencies with Npm'
         run: |
-          npm i -g npm@8.19.4
+          npm i -g npm@^10
           npm ci
 
       - name: 'Run unit tests'
@@ -41,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Build Docker Image'
         run: |

.github/workflows/develop-func-deploy.yml

@@ -33,6 +33,10 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.DEV_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.DEV_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.DEV_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.DEV_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   should_run:
@@ -44,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1.4.7
@@ -138,6 +142,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 
@@ -166,7 +190,7 @@ jobs:
         run: sleep 180
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Install newman'
         run: |

.github/workflows/prod-func-deploy.yml

@@ -23,13 +23,17 @@ env:
   ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
   KV_URI: ${{ secrets.PROD_KV_URI }}
   BASIC_GITHUB_TOKEN: ${{ secrets.BASIC_GITHUB_TOKEN }}
+  DATASTORE_SERVICES_URL: ${{ secrets.PROD_DATASTORE_SERVICES_URL }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}
+  DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: ${{ secrets.PROD_DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}
+  DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: ${{ secrets.PROD_DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}
 
 jobs:
   build-test-deploy:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Login via Azure CLI'
         uses: azure/login@v1.4.7
@@ -123,6 +127,26 @@ jobs:
                     "name": "languageWorkers__node__arguments",
                     "value": "--max-old-space-size=6656",
                     "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_URL",
+                    "value": "${{ env.DATASTORE_SERVICES_URL }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE",
+                    "value": "${{ env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE }}",
+                    "slotSetting": false
+                },
+                {
+                    "name": "DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS",
+                    "value": "${{ env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS }}",
+                    "slotSetting": false
                 }
             ]
 
@@ -151,7 +175,7 @@ jobs:
         run: sleep 300
 
       - name: 'Checkout GitHub Action'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 'Install newman'
         run: |

.nvmrc

@@ -1 +1 @@
-lts/gallium
+lts/iron

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/azure-functions/node:4.7.2.1-node16
+FROM mcr.microsoft.com/azure-functions/node:4.33.1-node20
 
 # install libxml2-utils for xmllint
 RUN \
@@ -14,6 +14,6 @@ COPY . /home/site/wwwroot
 # Install node_modules 
 WORKDIR /home/site/wwwroot
 RUN \
-    npm i -g npm@8.19.4 && \
+    npm i -g npm@^10 && \
     npm pkg delete scripts.prepare && \
     npm ci --production

README.md

@@ -1,3 +1,16 @@
+Summary
+=======
+
+Product  |  Validator API Endpoints
+--- | ---
+Description | Node.JS app that provides API end points to validate IATI XML files, is used by https://validator.iatistandard.org/
+Website |  [https://developer.iatistandard.org/](https://developer.iatistandard.org/)
+Related | [IATI/validator-services](https://github.com/IATI/validator-services), [IATI/validator-web](https://github.com/IATI/validator-web)
+Documentation | [https://developer.iatistandard.org/](https://developer.iatistandard.org/)
+Technical Issues | https://github.com/IATI/js-validator-api/issues
+Support | https://iatistandard.org/en/guidance/get-support/
+
+
 # IATI JavaScript Validator API
 
 [![Deploy_To_Dev_Function_On_Push](https://github.com/IATI/js-validator-api/actions/workflows/develop-func-deploy.yml/badge.svg)](https://github.com/IATI/js-validator-api/actions/workflows/develop-func-deploy.yml)
@@ -48,7 +61,7 @@ This should return a version number (change port if running directly instead of
 
 To test the validation is working, obtain an IATI XML file, and run the following command (this is if running via the docker setup; change port to `7071` if running directly):
 
-`curl -X POST -d @PATH_TO_IATI_XML http://localhost:8080/api/pub/validate`
+`curl -X POST --data-binary @PATH_TO_IATI_XML http://localhost:8080/api/pub/validate`
 
 You should see something like (if the file is a valid IATI file):
 
@@ -99,6 +112,14 @@ VALIDATOR_SERVICES_KEY_VALUE=
 
 -   URL and API Key for Validator Services, used to get list of Publisher Identifiers
 
+DATASTORE_SERVICES_URL=https://dev-func-datastore-services.azurewebsites.net/api
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME=x-functions-key
+DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE=
+DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS=5000
+
+- URL and API Key for datastore services, used by the advisory system to check for the
+  existence of IATI Identifiers in the Datastore
+
 ### App config defaults (set in `config/config.js`)
 
 ```

advisory-definitions/advisoryDefinitions-2.01.json

@@ -0,0 +1,32 @@
+[   
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]

advisory-definitions/advisoryDefinitions-2.02.json

@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]

advisory-definitions/advisoryDefinitions-2.03.json

@@ -0,0 +1,62 @@
+[
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/participating-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "activity-id",
+        "id": "1000.0.1",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in participating-org/@activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.2",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/provider-org/@provider-activity-id should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/planned-disbursement/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.3",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the planned-disbursement/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },     
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/provider-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "provider-activity-id",
+        "id": "1000.0.4",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/provider-org/@provider-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/transaction/receiver-org",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "receiver-activity-id",
+        "id": "1000.0.5",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The linked activity ID in the transaction/receiver-org/@receiver-activity-id attribute should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    },
+    {
+        "advisoryXPath": "/iati-activities/iati-activity/related-activity",
+        "advisoryTestName": "iati_identifier_exists",
+        "attributeToTest": "ref",
+        "id": "1000.0.6",
+        "severity": "advisory",
+        "category": "identifiers",
+        "message": "The related activity ID in related-activity/@ref should exist in the file being validated or in the IATI Datastore.",
+        "link": ""
+    }
+]

config/config.js

@@ -19,6 +19,10 @@ const config = {
     VALIDATOR_SERVICES_URL: process.env.VALIDATOR_SERVICES_URL,
     VALIDATOR_SERVICES_KEY_NAME: process.env.VALIDATOR_SERVICES_KEY_NAME,
     VALIDATOR_SERVICES_KEY_VALUE: process.env.VALIDATOR_SERVICES_KEY_VALUE,
+    DATASTORE_SERVICES_URL: process.env.DATASTORE_SERVICES_URL,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_NAME,
+    DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE: process.env.DATASTORE_SERVICES_AUTH_HTTP_HEADER_VALUE,
+    DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS: process.env.DATASTORE_SERVICES_IATI_IDENTIFIERS_EXIST_MAX_NUMBER_OF_IDS
 };
 
 export default config;