diff --git a/lib/bin/purge.js b/lib/bin/purge.js index 20909d063..2fa01e520 100644 --- a/lib/bin/purge.js +++ b/lib/bin/purge.js @@ -21,11 +21,13 @@ const { purgeTask } = require('../task/purge'); const { program } = require('commander'); program.option('-f, --force', 'Force any soft-deleted form to be purged right away.'); -program.option('-m, --mode ', 'Mode of purging. Can be "forms", "submissions", or "all". Default is "all".', 'all'); +program.option('-m, --mode ', 'Mode of purging. Can be "forms", "submissions", "entities" or "all". Default is "all".', 'all'); program.option('-i, --formId ', 'Purge a specific form based on its id.', parseInt); program.option('-p, --projectId ', 'Restrict purging to a specific project.', parseInt); program.option('-x, --xmlFormId ', 'Restrict purging to specific form based on xmlFormId (must be used with projectId).'); program.option('-s, --instanceId ', 'Restrict purging to a specific submission based on instanceId (use with projectId and xmlFormId).'); +program.option('-d, --datasetName ', 'Restrict purging to specific dataset/entity-list based on its name (must be used with projectId).'); +program.option('-e, --entityUuid ', 'Restrict purging to a specific entitiy based on its UUID (use with projectId and datasetName).'); program.parse(); diff --git a/lib/model/migrations/20241224-02-cascade-entity-purge.js b/lib/model/migrations/20241224-02-cascade-entity-purge.js new file mode 100644 index 000000000..7d39083c2 --- /dev/null +++ b/lib/model/migrations/20241224-02-cascade-entity-purge.js @@ -0,0 +1,20 @@ +// Copyright 2024 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +const up = (db) => db.raw(` +ALTER TABLE public.entity_defs DROP CONSTRAINT entity_defs_entityid_foreign; +ALTER TABLE public.entity_defs ADD CONSTRAINT entity_defs_entityid_foreign FOREIGN KEY ("entityId") REFERENCES public.entities(id) ON DELETE CASCADE; +`); + +const down = ((db) => db.raw(` +ALTER TABLE public.entity_defs DROP CONSTRAINT entity_defs_entityid_foreign; +ALTER TABLE public.entity_defs ADD CONSTRAINT entity_defs_entityid_foreign FOREIGN KEY ("entityId") REFERENCES public.entities(id); +`)); + +module.exports = { up, down }; diff --git a/lib/model/migrations/20241226-01-indices-for-purging-entities.js b/lib/model/migrations/20241226-01-indices-for-purging-entities.js new file mode 100644 index 000000000..edafa84a3 --- /dev/null +++ b/lib/model/migrations/20241226-01-indices-for-purging-entities.js @@ -0,0 +1,24 @@ +// Copyright 2024 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +const up = (db) => db.raw(` +CREATE INDEX audits_details_entity_uuid ON public.audits USING hash ((details->'entity'->>'uuid')) +WHERE ACTION IN ('entity.create', 'entity.update', 'entity.update.version', 'entity.update.resolve', 'entity.delete', 'entity.restore'); + +CREATE INDEX audits_details_entityUuids ON audits USING gin ((details -> 'entityUuids') jsonb_path_ops) +WHERE ACTION = 'entities.purge'; +`); + +const down = ((db) => db.raw(` +DROP INDEX audits_details_entity_uuid; +DROP INDEX audits_details_entityUuids; +`)); + +module.exports = { up, down }; + diff --git a/lib/model/migrations/20241227-01-backfill-audit-entity-uuid.js b/lib/model/migrations/20241227-01-backfill-audit-entity-uuid.js new file mode 100644 index 000000000..47ff576a5 --- /dev/null +++ b/lib/model/migrations/20241227-01-backfill-audit-entity-uuid.js @@ -0,0 +1,17 @@ +// Copyright 2024 ODK Central Developers +// See the NOTICE file at the top-level directory of this distribution and at +// https://github.com/getodk/central-backend/blob/master/NOTICE. +// This file is part of ODK Central. It is subject to the license terms in +// the LICENSE file found in the top-level directory of this distribution and at +// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central, +// including this file, may be copied, modified, propagated, or distributed +// except according to the terms contained in the LICENSE file. + +const up = (db) => db.raw(` +UPDATE audits SET details = ('{ "entity":{ "uuid":"' || (details->>'uuid') || '"}}')::JSONB +WHERE action = 'entity.delete' AND details \\? 'uuid'; +`); + +const down = () => {}; + +module.exports = { up, down }; diff --git a/lib/model/query/entities.js b/lib/model/query/entities.js index 89fa27782..ae6b2cc46 100644 --- a/lib/model/query/entities.js +++ b/lib/model/query/entities.js @@ -20,6 +20,18 @@ const { isTrue } = require('../../util/http'); const Problem = require('../../util/problem'); const { getOrReject, runSequentially } = require('../../util/promise'); +///////////////////////////////////////////////////////////////////////////////// +// Check if provided UUIDs (array) were used by purged entities. + +const _getPurgedUuids = (all, uuids) => all(sql` + SELECT jsonb_array_elements_text(details -> 'entityUuids') AS uuid FROM audits a WHERE action = 'entities.purge' + INTERSECT + SELECT jsonb_array_elements_text(${JSON.stringify(uuids)})`) + .then(all.map(r => r.uuid)); + +const _isPurgedUuid = (maybeOneFirst, uuid) => maybeOneFirst(sql` + SELECT TRUE FROM audits WHERE action = 'entities.purge' AND details -> 'entityUuids' @> ${JSON.stringify([uuid])} + `); ///////////////////////////////////////////////////////////////////////////////// // ENTITY DEF SOURCES @@ -65,10 +77,16 @@ const nextval = sql`nextval(pg_get_serial_sequence('entities', 'id'))`; // standard authenticated API request. The worker has better access to the event // actor/initiator and actee/target so it will do the logging itself (including // error logging). -const createNew = (dataset, partial, subDef, sourceId, userAgentIn) => ({ one, context }) => { +const createNew = (dataset, partial, subDef, sourceId, userAgentIn) => async ({ one, context, maybeOneFirst }) => { let creatorId; let userAgent; + // Validate UUID against purged entities + (await _isPurgedUuid(maybeOneFirst, partial.uuid)) + .ifDefined(() => { + throw Problem.user.uniquenessViolation({ fields: 'uuid', values: partial.uuid }); + }); + // Set creatorId and userAgent from submission def if it exists if (subDef != null) { ({ submitterId: creatorId, userAgent } = subDef); @@ -111,6 +129,12 @@ const createMany = (dataset, rawEntities, sourceId, userAgentIn) => async ({ all const creatorId = context.auth.actor.map((actor) => actor.id).orNull(); const userAgent = blankStringToNull(userAgentIn); + // Validate UUID with the purged entities + const purgedUuids = await _getPurgedUuids(all, rawEntities.map(e => e.uuid)); + if (purgedUuids.length > 0) { + throw Problem.user.uniquenessViolation({ fields: 'uuid', values: purgedUuids.join(', ') }); + } + // Augment parsed entity data with dataset and creator IDs const entitiesForInsert = rawEntities.map(e => new Entity({ datasetId: dataset.id, creatorId, ...e })); const entities = await all(sql`${insertMany(entitiesForInsert)} RETURNING id`); @@ -849,7 +873,12 @@ CROSS JOIN const del = (entity) => ({ run }) => run(markDeleted(entity)); -del.audit = (entity, dataset) => (log) => log('entity.delete', entity.with({ acteeId: dataset.acteeId }), { uuid: entity.uuid }); +del.audit = (entity, dataset) => (log) => + log( + 'entity.delete', + entity.with({ acteeId: dataset.acteeId }), + { entity: { uuid: entity.uuid } } + ); //////////////////////////////////////////////////////////////////////////////// // RESTORE ENTITY @@ -857,7 +886,81 @@ del.audit = (entity, dataset) => (log) => log('entity.delete', entity.with({ act const restore = (entity) => ({ run }) => run(markUndeleted(entity)); -restore.audit = (entity, dataset) => (log) => log('entity.restore', entity.with({ acteeId: dataset.acteeId }), { uuid: entity.uuid }); +restore.audit = (entity, dataset) => (log) => + log( + 'entity.restore', + entity.with({ acteeId: dataset.acteeId }), + { entity: { uuid: entity.uuid } } + ); + +//////////////////////////////////////////////////////////////////////////////// +// PURGE DELETED ENTITIES + +const PURGE_DAY_RANGE = config.has('default.taskSchedule.purge') + ? config.get('default.taskSchedule.purge') + : 30; // Default is 30 days + +const _trashedFilter = (force, projectId, datasetName, entityUuid) => { + const idFilters = [sql`TRUE`]; + if (projectId) idFilters.push(sql`datasets."projectId" = ${projectId}`); + if (datasetName) idFilters.push(sql`datasets."name" = ${datasetName}`); + if (entityUuid) idFilters.push(sql`entities."uuid" = ${entityUuid}`); + + const idFilter = sql.join(idFilters, sql` AND `); + + return (force + ? sql`entities."deletedAt" IS NOT NULL AND ${idFilter}` + : sql`entities."deletedAt" < CURRENT_DATE - CAST(${PURGE_DAY_RANGE} AS INT) AND ${idFilter}`); +}; + +const purge = (force = false, projectId = null, datasetName = null, entityUuid = null) => ({ oneFirst }) => { + if (entityUuid && (!projectId || !datasetName)) { + throw Problem.internal.unknown({ error: 'Must specify projectId and datasetName to purge a specify entity.' }); + } + if (datasetName && !projectId) { + throw Problem.internal.unknown({ error: 'Must specify projectId to purge all entities of a dataset/entity-list.' }); + } + + // Reminder: 'notes' in audit table is set by 'x-action-notes' header of all + // APIs with side-effects. Although we don't provide any way to set it from + // the frontend (as of 2024.3), it is a good idea to clear it for purged + // entities. + return oneFirst(sql` + WITH redacted_audits AS ( + UPDATE audits SET notes = NULL + FROM entities + JOIN datasets ON entities."datasetId" = datasets.id + WHERE (audits.details->'entity'->>'uuid') = entities.uuid + AND ${_trashedFilter(force, projectId, datasetName, entityUuid)} + ), purge_audit AS ( + INSERT INTO audits ("action", "loggedAt", "processed", "details") + SELECT 'entities.purge', clock_timestamp(), clock_timestamp(), jsonb_build_object('entitiesDeleted', COUNT(*), 'entityUuids', ARRAY_AGG(uuid)) + FROM entities + JOIN datasets ON entities."datasetId" = datasets.id + WHERE ${_trashedFilter(force, projectId, datasetName, entityUuid)} + HAVING count(*) > 0 + ), delete_entity_sources AS ( + DELETE FROM entity_def_sources + USING entity_defs, entities, datasets + WHERE entity_def_sources.id = entity_defs."sourceId" + AND entity_defs."entityId" = entities.id + AND entities."datasetId" = datasets.id + AND ${_trashedFilter(force, projectId, datasetName, entityUuid)} + ), delete_submission_backlog AS ( + DELETE FROM entity_submission_backlog + USING entities, datasets + WHERE entity_submission_backlog."entityUuid"::varchar = entities.uuid + AND entities."datasetId" = datasets.id + AND ${_trashedFilter(force, projectId, datasetName, entityUuid)} + ), deleted_entities AS ( + DELETE FROM entities + USING datasets + WHERE entities."datasetId" = datasets.id + AND ${_trashedFilter(force, projectId, datasetName, entityUuid)} + RETURNING 1 + ) + SELECT COUNT(*) FROM deleted_entities`); +}; module.exports = { createNew, _processSubmissionEvent, @@ -875,5 +978,5 @@ module.exports = { countByDatasetId, getById, getDef, getAll, getAllDefs, del, createEntitiesFromPendingSubmissions, - resolveConflict, restore + resolveConflict, restore, purge }; diff --git a/lib/task/purge.js b/lib/task/purge.js index 9bc0c819b..fa96ecf74 100644 --- a/lib/task/purge.js +++ b/lib/task/purge.js @@ -12,8 +12,12 @@ const { task } = require('./task'); const purgeTask = task.withContainer((container) => async (options = {}) => { // Form/submission purging will happen within its own transaction const message = await container.db.transaction(async trxn => { - const { Forms, Submissions } = container.with({ db: trxn }); + const { Forms, Submissions, Entities } = container.with({ db: trxn }); try { + if (options.mode === 'entities' || options.datasetName || options.entityUuid) { + const count = await Entities.purge(options.force, options.projectId, options.datasetName, options.entityUuid); + return `Entities purged: ${count}`; + } if (options.mode === 'submissions' || options.instanceId) { const count = await Submissions.purge(options.force, options.projectId, options.xmlFormId, options.instanceId); return `Submissions purged: ${count}`; @@ -21,14 +25,15 @@ const purgeTask = task.withContainer((container) => async (options = {}) => { const count = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId); return `Forms purged: ${count}`; } else { - // Purge both Forms and Submissions according to options + // Purge all Forms, Submissions and Entities according to options const formCount = await Forms.purge(options.force, options.formId, options.projectId, options.xmlFormId); const submissionCount = await Submissions.purge(options.force, options.projectId, options.xmlFormId, options.instanceId); + const entitiesCount = await Entities.purge(options.force, options.projectId, options.datasetName, options.entityUuid); // Related to form purging: deletes draft form defs that are not in use by any form and have no associated submission defs await Forms.clearUnneededDrafts(); - return `Forms purged: ${formCount}, Submissions purged: ${submissionCount}`; + return `Forms purged: ${formCount}, Submissions purged: ${submissionCount}, Entities purged: ${entitiesCount}`; } } catch (error) { return error?.problemDetails?.error; diff --git a/test/integration/api/entities.js b/test/integration/api/entities.js index 101dfe920..7841a1ecf 100644 --- a/test/integration/api/entities.js +++ b/test/integration/api/entities.js @@ -2105,7 +2105,7 @@ describe('Entities API', () => { .then(o => o.get()) .then(audit => { audit.acteeId.should.not.be.null(); - audit.details.uuid.should.be.eql('12345678-1234-4123-8234-123456789abc'); + audit.details.entity.uuid.should.be.eql('12345678-1234-4123-8234-123456789abc'); }); await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc') @@ -2169,7 +2169,7 @@ describe('Entities API', () => { .then(o => o.get()) .then(audit => { audit.acteeId.should.not.be.null(); - audit.details.uuid.should.be.eql('12345678-1234-4123-8234-123456789abc'); + audit.details.entity.uuid.should.be.eql('12345678-1234-4123-8234-123456789abc'); }); await asAlice.get('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc') diff --git a/test/integration/other/entities-purging.js b/test/integration/other/entities-purging.js new file mode 100644 index 000000000..57b247ddb --- /dev/null +++ b/test/integration/other/entities-purging.js @@ -0,0 +1,441 @@ +const should = require('should'); +const { sql } = require('slonik'); +const { testService } = require('../setup'); +const testData = require('../../data/xml'); +const { v4: uuid } = require('uuid'); + +const appPath = require('app-root-path'); +const Problem = require('../../../lib/util/problem'); +const { exhaust } = require(appPath + '/lib/worker/worker'); + +const createProject = (user) => user.post('/v1/projects') + .send({ name: 'a project ' + new Date().getTime() }) + .expect(200) + .then(({ body: project }) => project.id); + +const createEntities = async (user, count, projectId, datasetName) => { + const uuids = []; + for (let i = 0; i < count; i += 1) { + const _uuid = uuid(); + // eslint-disable-next-line no-await-in-loop + await user.post(`/v1/projects/${projectId}/datasets/${datasetName}/entities`) + .send({ + uuid: _uuid, + label: 'John Doe' + }) + .expect(200); + uuids.push(_uuid); + } + return uuids; +}; + +const createEntitiesViaSubmissions = async (user, container, count) => { + const uuids = []; + for (let i = 0; i < count; i += 1) { + const _uuid = uuid(); + // eslint-disable-next-line no-await-in-loop + await user.post('/v1/projects/1/forms/simpleEntity/submissions') + .send(testData.instances.simpleEntity.one + .replace(/one/g, `submission${i}`) + .replace(/88/g, i + 1) + .replace('uuid:12345678-1234-4123-8234-123456789abc', _uuid)) + .set('Content-Type', 'application/xml') + .expect(200); + uuids.push(_uuid); + } + await exhaust(container); + return uuids; +}; + +const deleteEntities = async (user, uuids, projectId, datasetName) => { + for (const _uuid of uuids) { + // eslint-disable-next-line no-await-in-loop + await user.delete(`/v1/projects/${projectId}/datasets/${datasetName}/entities/${_uuid}`) + .expect(200); + } +}; + +const createDataset = (user, projectId, name) => + user.post(`/v1/projects/${projectId}/datasets`) + .send({ name }); + +const createDeletedEntities = async (user, count, { datasetName='people', project = 1 } = {}) => { + await createDataset(user, project, datasetName); + const uuids = await createEntities(user, count, project, datasetName); + await deleteEntities(user, uuids, project, datasetName); + return uuids; +}; + +describe('query module entities purge', () => { + + describe('entities purge arguments', () => { + it('should purge a specific entity', testService(async (service, { Entities, oneFirst }) => { + const asAlice = await service.login('alice'); + + const uuids = await createDeletedEntities(asAlice, 2); + + // Purge Entities here should not purge anything because they were in the trash less than 30 days + let purgeCount = await Entities.purge(); + purgeCount.should.equal(0); + + // But we should be able to force purge an Entity + // specified by projectId, datasetName and uuid + purgeCount = await Entities.purge(true, 1, 'people', uuids[0]); + purgeCount.should.equal(1); + + // One (soft-deleted) entity should still be in the database + const entityCount = await oneFirst(sql`select count(*) from entities`); + entityCount.should.equal(1); + })); + + it('should purge deleted entities, which were created via submissions', testService(async (service, container) => { + const { Entities, oneFirst } = container; + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + const uuids = await createEntitiesViaSubmissions(asAlice, container, 2); + + await deleteEntities(asAlice, uuids, 1, 'people'); + + const purgeCount = await Entities.purge(true, 1, 'people', uuids[0]); + purgeCount.should.equal(1); + + // One (soft-deleted) entity should still be in the database + const entityCount = await oneFirst(sql`select count(*) from entities`); + entityCount.should.equal(1); + })); + + it('should purge all deleted entities of a dataset', testService(async (service, { Entities, all }) => { + const asAlice = await service.login('alice'); + + await createDeletedEntities(asAlice, 2, { datasetName: 'people' }); + const treeUuids = await createDeletedEntities(asAlice, 2, { datasetName: 'trees' }); + + const purgeCount = await Entities.purge(true, 1, 'people'); + purgeCount.should.equal(2); + + // 'trees' deleted entities are still there + const remainingEntities = await all(sql`select uuid from entities`); + remainingEntities.map(e => e.uuid).should.containDeep(treeUuids); + })); + + it('should purge all deleted entities under a project', testService(async (service, { Entities, all }) => { + const asAlice = await service.login('alice'); + + await createDeletedEntities(asAlice, 2, { datasetName: 'people', project: 1 }); + await createDeletedEntities(asAlice, 2, { datasetName: 'trees', project: 1 }); + + const secondProjectId = await createProject(asAlice); + const catsUuids = await createDeletedEntities(asAlice, 2, { datasetName: 'cats', project: secondProjectId }); + const dogsUuids = await createDeletedEntities(asAlice, 2, { datasetName: 'dogs', project: secondProjectId }); + + // purging all deleted entities of project 1 + const purgeCount = await Entities.purge(true, 1); + purgeCount.should.equal(4); + + const remainingEntities = await all(sql`select uuid from entities`); + remainingEntities.map(e => e.uuid).should.containDeep([...catsUuids, ...dogsUuids]); + })); + + it('should purge all deleted entities under a project', testService(async (service, { Entities, oneFirst }) => { + const asAlice = await service.login('alice'); + + await createDeletedEntities(asAlice, 2, { datasetName: 'people', project: 1 }); + await createDeletedEntities(asAlice, 2, { datasetName: 'trees', project: 1 }); + + const secondProjectId = await createProject(asAlice); + await createDeletedEntities(asAlice, 2, { datasetName: 'cats', project: secondProjectId }); + await createDeletedEntities(asAlice, 2, { datasetName: 'dogs', project: secondProjectId }); + + // purging all deleted entities + const purgeCount = await Entities.purge(true); + purgeCount.should.equal(8); + + // nothing should be there + const entityCount = await oneFirst(sql`select count(*) from entities`); + entityCount.should.equal(0); + })); + + const PROVIDE_ALL = 'Must specify projectId and datasetName to purge a specify entity.'; + const PROVIDE_PROJECT_ID = 'Must specify projectId to purge all entities of a dataset/entity-list.'; + const cases = [ + { description: ' when entityUuid specified without projectId and datasetName', + projectId: false, datasetName: false, entityUuid: true, expectedError: PROVIDE_ALL }, + { description: ' when entityUuid specified without projectId', + projectId: false, datasetName: true, entityUuid: true, expectedError: PROVIDE_ALL }, + { description: ' when entityUuid specified without datasetName', + projectId: true, datasetName: false, entityUuid: true, expectedError: PROVIDE_ALL }, + { description: ' when datasetName specified without projectId', + projectId: false, datasetName: true, entityUuid: false, expectedError: PROVIDE_PROJECT_ID }, + ]; + cases.forEach(c => + it(`should throw an error ${c.description}`, testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + + const uuids = await createDeletedEntities(asAlice, 1); + + (() => { + Entities.purge( + true, + c.projectId ? 1 : null, + c.datasetName ? 'people' : null, + c.entityUuid ? uuids[0] : null + ); + }).should.throw(Problem.internal.unknown({ + error: c.expectedError + })); + })) + ); + }); + + describe('30 day time limit', () => { + it('should purge multiple entities deleted over 30 days ago', testService(async (service, { Entities, all, run }) => { + const asAlice = await service.login('alice'); + + await createDeletedEntities(asAlice, 2); + + // Mark two as deleted a long time ago + await run(sql`update entities set "deletedAt" = '1999-1-1' where "deletedAt" is not null`); + + // More recent delete, within 30 day window + const recentUuids = await createDeletedEntities(asAlice, 2); + + const purgeCount = await Entities.purge(); + purgeCount.should.equal(2); + + // Recently deleted entities are not purged + const remainingEntities = await all(sql`select uuid from entities`); + remainingEntities.map(e => e.uuid).should.containDeep(recentUuids); + })); + + it('should purge recently deleted entities when forced', testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + + await createDeletedEntities(asAlice, 2); + + const purgeCount = await Entities.purge(); + purgeCount.should.equal(0); + })); + }); + + describe('deep cleanup of all submission artifacts', () => { + + it('should purge all versions of a deleted submission', testService(async (service, { Entities, all }) => { + const asAlice = await service.login('alice'); + + await createDataset(asAlice, 1, 'people'); + + const uuids = await createEntities(asAlice, 2, 1, 'people'); + + // Edit the Entity + await asAlice.patch(`/v1/projects/1/datasets/people/entities/${uuids[0]}?baseVersion=1`) + .send({ label: 'edited' }) + .expect(200); + + // Just delete the first Entity + await deleteEntities(asAlice, [uuids[0]], 1, 'people'); + + // Purge the Entities + const purgedCount = await Entities.purge(true); + purgedCount.should.be.eql(1); + + // Check that the entity is deleted + const remainingEntity = await all(sql`select * from entities`); + remainingEntity.map(e => e.uuid).should.containDeep([uuids[1]]); + + // Check that entity defs are also deleted + const remainingEntityDefs = await all(sql`select * from entity_defs`); + remainingEntityDefs.map(def => def.entityId).should.containDeep(remainingEntity.map(e => e.id)); + })); + + it('should redact notes of a deleted entity sent with x-action-notes', testService(async (service, { Entities, oneFirst }) => { + const asAlice = await service.login('alice'); + + // Create a dataset + await createDataset(asAlice, 1, 'people'); + + // Create an entity with action notes + const _uuid = uuid(); + await asAlice.post(`/v1/projects/1/datasets/people/entities`) + .send({ uuid: _uuid, label: 'John Doe' }) + .set('X-Action-Notes', 'a note about the entity') + .expect(200); + + // Check that the note exists in the entity's audit log + await asAlice.get(`/v1/projects/1/datasets/people/entities/${_uuid}/audits`) + .expect(200) + .then(({ body }) => { + body.length.should.equal(1); + body[0].notes.should.equal('a note about the entity'); + }); + + // Delete the entity + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${_uuid}`); + + // Purge the entity + await Entities.purge(true); + + // Look at what is in the audit log via the database because the entity is deleted + const auditNotes = await oneFirst(sql`select notes from audits where action = 'entity.create'`); + + // Check that the note is redacted + should(auditNotes).be.null(); + })); + + it('should purge entity sources', testService(async (service, container) => { + const { Entities, oneFirst } = container; + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + const uuids = await createEntitiesViaSubmissions(asAlice, container, 2); + + let sourcesCount = await oneFirst(sql`select count(1) from entity_def_sources`); + sourcesCount.should.be.equal(2); + + await deleteEntities(asAlice, uuids, 1, 'people'); + + await Entities.purge(true); + + sourcesCount = await oneFirst(sql`select count(1) from entity_def_sources`); + sourcesCount.should.be.equal(0); + })); + + it('should purge submission backlog for entities', testService(async (service, container) => { + const { Entities, oneFirst } = container; + const asAlice = await service.login('alice'); + + // Publish a form that will set up the dataset with properties + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.offlineEntity) + .set('Content-Type', 'application/xml') + .expect(200); + + const uuids = ['12345678-1234-4123-8234-123456789abc']; + + // Create an entity via the API (to be updated offline) + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: uuids[0], + label: 'Johnny Doe', + data: { first_name: 'Johnny', age: '22' } + }) + .expect(200); + + await exhaust(container); + + const branchId = uuid(); + + // Send second update in first + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${branchId}"`) + .replace('one', 'one-update1') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('arrived', 'working') + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + let backlogCount = await oneFirst(sql`select count(1) from entity_submission_backlog`); + backlogCount.should.be.equal(1); + + await deleteEntities(asAlice, uuids, 1, 'people'); + + await Entities.purge(true); + + backlogCount = await oneFirst(sql`select count(1) from entity_submission_backlog`); + backlogCount.should.be.equal(0); + })); + }); + + describe('restrict recreation with same uuid', () => { + it('should not allow reuse of purged uuid when creating single entity', testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + + const uuids = await createDeletedEntities(asAlice, 2); + + const purgeCount = await Entities.purge(true); + purgeCount.should.equal(2); + + await asAlice.post(`/v1/projects/1/datasets/people/entities`) + .send({ + uuid: uuids[0], + label: 'John Doe' + }) + .expect(409) + .then(({ body }) => { + body.details.values.should.eql(uuids[0]); + }); + })); + + it('should not allow reuse of purged uuid when creating bulk entities', testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + + const uuids = await createDeletedEntities(asAlice, 2); + + const purgeCount = await Entities.purge(true); + purgeCount.should.equal(2); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + source: { + name: 'people.csv', + size: 100, + }, + entities: [ + { + uuid: uuids[0], + label: 'Johnny Doe' + }, + { + uuid: uuids[1], + label: 'Alice' + }, + ] + }) + .expect(409) + .then(({ body }) => { + uuids.forEach(i => body.details.values.includes(i).should.be.true); + }); + })); + }); + + describe('entity.purge audit event', () => { + it('should log a purge event in the audit log when purging submissions', testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + + const uuids = await createDeletedEntities(asAlice, 2); + + // Purge entities + await Entities.purge(true); + + await asAlice.get('/v1/audits') + .then(({ body }) => { + body.filter((a) => a.action === 'entities.purge').length.should.equal(1); + body[0].details.entitiesDeleted.should.eql(2); + body[0].details.entityUuids.should.containDeep(uuids); + }); + })); + + it('should not log event if no entities purged', testService(async (service, { Entities }) => { + const asAlice = await service.login('alice'); + // No deleted entities exist here to purge + await Entities.purge(true); + + await asAlice.get('/v1/audits') + .then(({ body }) => { + body.filter((a) => a.action === 'entities.purge').length.should.equal(0); + }); + })); + }); +}); diff --git a/test/integration/other/migrations.js b/test/integration/other/migrations.js index b582c83da..31f136cd5 100644 --- a/test/integration/other/migrations.js +++ b/test/integration/other/migrations.js @@ -1237,3 +1237,55 @@ testMigration('20240914-02-remove-orphaned-client-audits.js', () => { })); }); }); + +testMigration('20241227-01-backfill-audit-entity-uuid.js', () => { + it('should update the format of detail for entity.delete audits', testServiceFullTrx(async (service, container) => { + await populateUsers(container); + await populateForms(container); + + const asAlice = await service.login('alice'); + + // Create a dataset + await asAlice.post('/v1/projects/1/datasets') + .send({ name: 'people' }); + + // Create an entity + const _uuid = uuid(); + await asAlice.post(`/v1/projects/1/datasets/people/entities`) + .send({ + uuid: _uuid, + label: 'John Doe' + }) + .expect(200); + + // Delete the entity + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${_uuid}`) + .expect(200); + + // Update audit log to match the previous code where uuid is written at the root + await container.run(sql`UPDATE audits SET details=${JSON.stringify({ uuid: _uuid })} WHERE action='entity.delete'`); + + // Check the details of audit log of entity.delete action + await asAlice.get('/v1/audits?action=entity.delete') + .expect(200) + .then(({ body }) => { + const [ audit ] = body; + audit.details.should.be.eql({ + uuid: _uuid + }); + }); + + // Run the migration + await up(); + + // Check the details of audit log of entity.delete action + await asAlice.get('/v1/audits?action=entity.delete') + .expect(200) + .then(({ body }) => { + const [ audit ] = body; + audit.details.should.be.eql({ + entity: { uuid: _uuid } + }); + }); + })); +}); diff --git a/test/integration/task/purge.js b/test/integration/task/purge.js index 8ac9a5a54..e5295666c 100644 --- a/test/integration/task/purge.js +++ b/test/integration/task/purge.js @@ -7,7 +7,9 @@ const { purgeTask } = require(appRoot + '/lib/task/purge'); // The basics of this task are tested here, including returning the message // of purged forms, but the full functionality is more thoroughly tested in -// test/integration/other/form-purging.js and test/integration/other/submission-purging.js. +// test/integration/other/form-purging.js, +// test/integration/other/submission-purging.js and +// test/integration/other/entities-purging.js const withDeleteChecks = container => { const confirm = { @@ -41,7 +43,7 @@ const withDeleteChecks = container => { const testPurgeTask = fn => testTask(container => fn(withDeleteChecks(container))); -describe('task: purge deleted resources (forms and submissions)', () => { +describe('task: purge deleted resources (forms, submissions and entities)', () => { describe('forms', () => { describe('force flag', () => { it('should not purge recently deleted forms by default', testPurgeTask(({ confirm, Forms }) => @@ -239,13 +241,57 @@ describe('task: purge deleted resources (forms and submissions)', () => { }))); }); + describe('entities', () => { + it('should call entities purge if mode is specified as entities', testTask(() => + purgeTask({ mode: 'entities' }) + .then((message) => { + message.should.equal('Entities purged: 0'); + }))); + + it('should call entities purge if entities uuid is specified', testTask(() => + purgeTask({ entityUuid: 'abc', projectId: 1, datasetName: 'people' }) + .then((message) => { + message.should.equal('Entities purged: 0'); + }))); + + it('should call entities purge if dataset name is specified', testTask(() => + purgeTask({ projectId: 1, datasetName: 'people' }) + .then((message) => { + message.should.equal('Entities purged: 0'); + }))); + + it('should complain if uuid specified without project and dataset', testTask(() => + purgeTask({ entityUuid: 'abc' }) + .then((message) => { + message.should.equal('Must specify projectId and datasetName to purge a specify entity.'); + }))); + + it('should complain if uuid specified without project', testTask(() => + purgeTask({ entityUuid: 'abc', datasetName: 'simple' }) + .then((message) => { + message.should.equal('Must specify projectId and datasetName to purge a specify entity.'); + }))); + + it('should complain if uuid specified without dataset', testTask(() => + purgeTask({ entityUuid: 'abc', projectId: 1 }) + .then((message) => { + message.should.equal('Must specify projectId and datasetName to purge a specify entity.'); + }))); + + it('should complain if dataset specified without project', testTask(() => + purgeTask({ datasetName: 'simple' }) + .then((message) => { + message.should.equal('Must specify projectId to purge all entities of a dataset/entity-list.'); + }))); + }); + describe('all', () => { it('should purge both forms and submissions when neither mode is specified (not forced)', testTask(({ Forms }) => Forms.getByProjectAndXmlFormId(1, 'simple') .then((form) => Forms.del(form.get()) .then(() => purgeTask()) .then((message) => { - message.should.equal('Forms purged: 0, Submissions purged: 0'); + message.should.equal('Forms purged: 0, Submissions purged: 0, Entities purged: 0'); })))); it('should purge both forms and submissions when neither mode is specified (forced)', testTask(({ Forms }) => @@ -253,7 +299,7 @@ describe('task: purge deleted resources (forms and submissions)', () => { .then((form) => Forms.del(form.get()) .then(() => purgeTask({ force: true })) .then((message) => { - message.should.equal('Forms purged: 1, Submissions purged: 0'); + message.should.equal('Forms purged: 1, Submissions purged: 0, Entities purged: 0'); })))); it('should accept other mode and treat as "all"', testTask(({ Forms }) => @@ -261,7 +307,7 @@ describe('task: purge deleted resources (forms and submissions)', () => { .then((form) => Forms.del(form.get()) .then(() => purgeTask({ force: true, mode: 'something_else' })) .then((message) => { - message.should.equal('Forms purged: 1, Submissions purged: 0'); + message.should.equal('Forms purged: 1, Submissions purged: 0, Entities purged: 0'); })))); }); });