feature #668: Entities integrity URL

lib/formats/openrosa.js

@@ -87,5 +87,16 @@ const openRosaErrorTemplate = openRosaMessageBase('error');
 parse(openRosaErrorTemplate);
 const openRosaError = (message) => render(openRosaErrorTemplate, { message });
 
-module.exports = { createdMessage, formList, formManifest, openRosaError };
+const entityListTemplate = template(200, `<?xml version="1.0" encoding="UTF-8"?>
+  <data>
+    <entities>
+    {{#entities}}
+      <entity id="{{uuid}}">
+        <deleted>{{deleted}}</deleted>
+      </entity>
+    {{/entities}}
+    </entities>
+  </data>`);
+const entityList = (data) => entityListTemplate(data);
+module.exports = { createdMessage, formList, formManifest, openRosaError, entityList };
 

lib/model/query/datasets.js

@@ -426,21 +426,21 @@ const getPublishedBySimilarName = (projectId, name) => ({ maybeOne }) => {
 ////////////////////////////////////////////////////////////////////////////////
 // DATASET METADATA GETTERS
 
+const _getLinkedForms = (datasetName, projectId) => sql`
+SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
+JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
+JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
+JOIN form_defs current_def ON f."currentDefId" = current_def.id
+JOIN datasets ds ON ds.id = fa."datasetId"
+WHERE ds.name = ${datasetName}
+  AND ds."projectId" = ${projectId}
+  AND ds."publishedAt" IS NOT NULL
+`;
+
 // Gets the dataset information, properties (including which forms each property comes from),
 // and which forms consume the dataset via CSV attachment.
 const getMetadata = (dataset) => async ({ all, Datasets }) => {
 
-  const _getLinkedForms = (datasetName, projectId) => sql`
-    SELECT DISTINCT f."xmlFormId", coalesce(current_def.name, f."xmlFormId") "name" FROM form_attachments fa
-    JOIN form_defs fd ON fd.id = fa."formDefId" AND fd."publishedAt" IS NOT NULL
-    JOIN forms f ON f.id = fd."formId" AND f."deletedAt" IS NULL
-    JOIN form_defs current_def ON f."currentDefId" = current_def.id
-    JOIN datasets ds ON ds.id = fa."datasetId"
-    WHERE ds.name = ${datasetName}
-      AND ds."projectId" = ${projectId}
-      AND ds."publishedAt" IS NOT NULL
-  `;
-
   const _getSourceForms = (datasetName, projectId) => sql`
     SELECT DISTINCT f."xmlFormId", coalesce(fd.name, f."xmlFormId") "name" FROM datasets ds
     JOIN dataset_form_defs dfd ON ds.id = dfd."datasetId"    
@@ -489,7 +489,6 @@ const getMetadata = (dataset) => async ({ all, Datasets }) => {
   };
 };
 
-
 ////////////////////////////////////////////////////////////////////////////
 // DATASET PROPERTY GETTERS
 
@@ -665,6 +664,28 @@ const getLastUpdateTimestamp = (datasetId) => ({ maybeOne }) =>
     .then((t) => t.orNull())
     .then((t) => (t ? t.loggedAt : null));
 
+
+const canReadForOpenRosa = (auth, datasetName, projectId) => ({ oneFirst }) => oneFirst(sql`
+      WITH linked_forms AS (
+        ${_getLinkedForms(datasetName, projectId)}
+      )
+      SELECT count(1) FROM linked_forms
+      INNER JOIN (
+        SELECT forms."xmlFormId" FROM forms
+        INNER JOIN projects ON projects.id=forms."projectId"
+        INNER JOIN (
+          SELECT "acteeId" FROM assignments
+          INNER JOIN (
+            SELECT id FROM roles WHERE verbs ? 'form.read' OR verbs ? 'open_form.read'
+          ) AS role ON role.id=assignments."roleId"
+          WHERE "actorId"=${auth.actor.map((actor) => actor.id).orElse(-1)}
+        ) AS assignment ON assignment."acteeId" IN ('*', 'form', projects."acteeId", forms."acteeId")
+        WHERE forms.state != 'closed'
+        GROUP BY forms."xmlFormId"
+      ) AS users_forms ON users_forms."xmlFormId" = linked_forms."xmlFormId"
+    `)
+  .then(count => count > 0);
+
 module.exports = {
   createPublishedDataset, createPublishedProperty,
   createOrMerge, publishIfExists,
@@ -674,5 +695,5 @@ module.exports = {
   getProperties, getFieldsByFormDefId,
   getDiff, update, countUnprocessedSubmissions,
   getUnprocessedSubmissions,
-  getLastUpdateTimestamp
+  getLastUpdateTimestamp, canReadForOpenRosa
 };

lib/model/query/entities.js

@@ -964,6 +964,25 @@ const purge = (force = false, projectId = null, datasetName = null, entityUuid =
     SELECT COUNT(*) FROM deleted_entities`);
 };
 
+////////////////////////////////////////////////////////////////////////////////
+// INTEGRITY CHECK
+
+const idFilter = (options) => {
+  const query = options.ifArg('id', ids => sql`uuid IN (${sql.join(ids.split(',').map(id => sql`${id.trim()}`), sql`, `)})`);
+  return query.sql ? query : sql`TRUE`;
+};
+
+const _getAllEntitiesState = (datasetId, options) => sql`
+  SELECT uuid, "deletedAt" IS NOT NULL as deleted 
+  FROM entities 
+  WHERE "datasetId" = ${datasetId} AND ${idFilter(options)}
+  -- union with purged
+  -- union with not approved
+`;
+
+const getEntitiesState = (datasetId, options = QueryOptions.none) =>
+  ({ all }) => all(_getAllEntitiesState(datasetId, options));
+
 module.exports = {
   createNew, _processSubmissionEvent,
   createSource,
@@ -980,5 +999,5 @@ module.exports = {
   countByDatasetId, getById, getDef,
   getAll, getAllDefs, del,
   createEntitiesFromPendingSubmissions,
-  resolveConflict, restore, purge
+  resolveConflict, restore, purge, getEntitiesState
 };

lib/resources/datasets.js

@@ -8,14 +8,15 @@
 // except according to the terms contained in the LICENSE file.
 
 const sanitize = require('sanitize-filename');
-const { getOrNotFound } = require('../util/promise');
+const { getOrNotFound, reject } = require('../util/promise');
 const { streamEntityCsv } = require('../data/entity');
 const { validateDatasetName, validatePropertyName } = require('../data/dataset');
 const { contentDisposition, success, withEtag } = require('../util/http');
 const { md5sum } = require('../util/crypto');
 const { Dataset } = require('../model/frames');
 const Problem = require('../util/problem');
 const { QueryOptions } = require('../util/db');
+const { entityList } = require('../formats/openrosa');
 
 module.exports = (service, endpoint) => {
   service.get('/projects/:id/datasets', endpoint(({ Projects, Datasets }, { auth, params, queryOptions }) =>
@@ -102,4 +103,20 @@ module.exports = (service, endpoint) => {
 
     return withEtag(serverEtag, csv);
   }));
+
+  service.get('/projects/:projectId/datasets/:name/integrity', endpoint.openRosa(async ({ Datasets, Entities }, { params, auth, queryOptions }) => {
+    const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound);
+
+    // Anyone with the verb `entity.list` or anyone with read access on a Form
+    // that consumes this dataset can call this endpoint.
+    const canAccessEntityList = await auth.can('entity.list', dataset);
+    if (!canAccessEntityList) {
+      await Datasets.canReadForOpenRosa(auth, params.name, params.projectId)
+        .then(canAccess => canAccess || reject(Problem.user.insufficientRights()));
+    }
+
+    const entities = await Entities.getEntitiesState(dataset.id, queryOptions.allowArgs('id'));
+
+    return entityList({ entities });
+  }));
 };

package.json

@@ -68,6 +68,7 @@
     "should": "~13",
     "streamtest": "~1.2",
     "supertest": "^6.3.3",
-    "tmp": "~0.2"
+    "tmp": "~0.2",
+    "xml2js": "^0.5.0"
   }
 }