From d7c830bf653c7cd44d1db9624792cfe4af880cdf Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Wed, 19 Jul 2023 16:57:20 -0400 Subject: [PATCH 01/10] added integration tests to capture requirement --- test/integration/api/odata.js | 100 +++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 497292f10..3b0692a67 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -573,7 +573,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1, 2 ]) { + for (const idx of [0, 1, 2]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; @@ -717,6 +717,70 @@ describe('api: /forms/:id.svc', () => { }); })))); + // nb: order of id and createdAt is not guaranteed to be same + // in test env, see submission id 134849 and 134850 + // 50 (at 873 ms) was created before 49 (at 874 ms) + it.skip('should limit Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10') + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + }); + })); + + it.skip('should ignore $skip when $skipToken is given', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skip=10') + .expect(200) + .then(({ body }) => { + + body.value[0].instanceName.should.be.eql('eleven'); + body.value[9].instanceName.should.be.eql('twenty'); + + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink + '&$skip=10') + .expect(200) + .then(({ body }) => { + + body.value[0].instanceName.should.be.eql('twenty one'); + body.value[9].instanceName.should.be.eql('thirty'); + + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + }); + })); + + it.skip('should limit and filter Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$filter=__system/reviewState eq 'rejected'`) + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$filter=__system/reviewState eq 'rejected'&$skipToken=${token}`); + }); + })); + it('should provide toplevel row count if requested', testService((service) => withSubmissions(service, (asAlice) => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$count=true') @@ -777,7 +841,7 @@ describe('api: /forms/:id.svc', () => { .then(() => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$filter=__system/submitterId eq 5') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1 ]) { + for (const idx of [0, 1]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; @@ -1366,6 +1430,36 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body['@odata.count'].should.equal(2); + }) + })); + + it.skip('should limit subtable results', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10') + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + __id: body.value[body.value.length - 1].__id, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$skipToken=${token}`); + }); + })); + + it.skip('should limit and filter subtable', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$filter=$root/Submission/__system/reviewState eq 'rejected'`) + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[body.value.length - 1].instanceId, + __id: body.value[body.value.length - 1].__id, + } + const token = btoa(JSON.stringify(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$filter=$root/Submission/__system/reviewState eq 'rejected'&$skipToken=${token}`); }); })); @@ -1832,7 +1926,7 @@ describe('api: /forms/:id.svc', () => { .then(() => asAlice.get('/v1/projects/1/forms/withrepeat/draft.svc/Submissions') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1, 2 ]) { + for (const idx of [0, 1, 2]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; From bea63f4177613394fce608a35d96ae17325fa06f Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Thu, 27 Jul 2023 16:08:44 -0400 Subject: [PATCH 02/10] Made changes for Submission OData need to add more unit tests --- lib/data/odata.js | 16 +-- lib/formats/odata.js | 85 ++++++++++++--- lib/http/endpoint.js | 2 +- lib/model/query/submissions.js | 23 +++- lib/resources/odata.js | 6 +- lib/util/db.js | 26 ++++- lib/util/odata.js | 9 +- lib/util/util.js | 14 +++ test/integration/api/odata-entities.js | 2 +- test/integration/api/odata.js | 142 +++++++++++++++---------- test/unit/data/odata.js | 64 +++++------ test/unit/formats/odata.js | 21 ++-- test/unit/util/util.js | 8 ++ 13 files changed, 283 insertions(+), 135 deletions(-) diff --git a/lib/data/odata.js b/lib/data/odata.js index 6aad6f4bc..038fd1b12 100644 --- a/lib/data/odata.js +++ b/lib/data/odata.js @@ -136,9 +136,9 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // on option.metadata even though they are not part of the form's own schema. // So rather than try to inject them into the xml transformation below, we just // formulate them herein advance: - if (!options.metadata || options.metadata.__id) { - root.__id = submission.instanceId; - } + // if (!options.metadata || options.metadata.__id) { + root.__id = submission.instanceId; + // } if (table === 'Submissions') { const systemObj = { submissionDate: submission.createdAt, @@ -175,7 +175,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi } // bail out without doing any work if we are encrypted. - if (encrypted === true) return resolve(result); + if (encrypted === true) return resolve({ data: result, instanceId: submission.instanceId }); // we keep a dataStack, so we build an appropriate nested structure overall, and // we can select the appropriate layer of that nesting at will. @@ -220,9 +220,9 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // create our new databag, push into result data, and set it as our result ptr. const bag = generateDataFrame(schemaStack); - if (!options.metadata || options.metadata.__id) { - bag.__id = hashId(schemaStack, submission.instanceId); - } + // if (!options.metadata || options.metadata.__id) { + bag.__id = hashId(schemaStack, submission.instanceId); + // } dataPtr[outname].push(bag); dataStack.push(bag); @@ -342,7 +342,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi if (schemaStack.hasExited()) { parser.reset(); - resolve(result); + resolve({ data: result, instanceId: submission.instanceId }); } } }, { xmlMode: true, decodeEntities: true }); diff --git a/lib/formats/odata.js b/lib/formats/odata.js index 11464e81f..1474052d8 100644 --- a/lib/formats/odata.js +++ b/lib/formats/odata.js @@ -23,6 +23,8 @@ const { sanitizeOdataIdentifier, without } = require('../util/util'); const { jsonDataFooter, extractOptions, nextUrlFor } = require('../util/odata'); const { submissionToOData, systemFields } = require('../data/odata'); const { SchemaStack } = require('../data/schema'); +const { QueryOptions } = require('../util/db'); + //////////////////////////////////////////////////////////////////////////////// // UTIL @@ -52,9 +54,10 @@ const extractPathContext = (subpath) => const extractPaging = (table, query) => { const parsedLimit = parseInt(query.$top, 10); const limit = Number.isNaN(parsedLimit) ? Infinity : parsedLimit; - const offset = parseInt(query.$skip, 10) || 0; + const offset = (!query.$skiptoken && parseInt(query.$skip, 10)) || 0; const shouldCount = isTrue(query.$count); - const result = { limit: max(0, limit), offset: max(0, offset), shouldCount }; + const skipToken = query.$skiptoken ? QueryOptions.parseSkiptoken(query.$skiptoken) : null; + const result = { limit: max(0, limit), offset: max(0, offset), shouldCount, skipToken }; return Object.assign(result, (table === 'Submissions') ? { doLimit: Infinity, doOffset: 0 } @@ -366,11 +369,12 @@ const edmxForEntities = (datasetName, properties) => { // originalUrl: String is the request URL; we need it as well to formulate response URLs. // query: Object is the Express Request query object indicating request querystring parameters. // inStream: Stream[Row] is the postgres Submissions rowstream. -const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, tableCount) => { +const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, tableCount, tableRemaining) => { // cache values we'll need repeatedly. const serviceRoot = getServiceRoot(originalUrl); - const { limit, offset, doLimit, doOffset, shouldCount } = extractPaging(table, query); + const { doLimit, doOffset, shouldCount, skipToken } = extractPaging(table, query); const options = extractOptions(query); + const isSubTable = table !== 'Submissions'; // make sure the target table actually exists. // TODO: now that this doesn't require schema computation, should we move it up @@ -379,25 +383,47 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t // write the header, then transform and stream each row. let counted = 0; + let lastInstanceId = null; + let lastRepeatId = null; + let added = 0; + let remainingItems = 0; + + // For Submissions table, it is true because cursor is handled at database level + let cursorPredicate = !isSubTable || !skipToken; + const parserStream = new Transform({ writableObjectMode: true, // we take a stream of objects from the db, but readableObjectMode: false, // we put out a stream of text. transform(row, _, done) { // per row, we do our asynchronous parsing, jam the result onto the // text resultstream, and call done to indicate that the row is processed. - submissionToOData(fields, table, row, options).then((data) => { + submissionToOData(fields, table, row, options).then(({ data, instanceId }) => { const parentIdProperty = data[0] ? Object.keys(data[0]).find(p => /^__.*-id$/.test(p)) : null; + // In case of subtable we are reading all Submissions without pagination because we have to + // count repeat items in each Submission for (const field of data) { // if $select is there and parentId is not requested then remove it - const fieldRefined = options.metadata && !options.metadata[parentIdProperty] ? without([parentIdProperty], field) : field; + let fieldRefined = options.metadata && !options.metadata[parentIdProperty] ? without([parentIdProperty], field) : field; + // if $select is there and __id is not requested then remove it + fieldRefined = options.metadata && !options.metadata.__id ? without(['__id'], fieldRefined) : fieldRefined; - if ((counted >= doOffset) && (counted < (doOffset + doLimit))) { - this.push((counted === doOffset) ? '{"value":[' : ','); // header or fencepost. + if (added === doLimit) remainingItems += 1; + + if (added < doLimit && counted >= doOffset && cursorPredicate) { + this.push((added === 0) ? '{"value":[' : ','); // header or fencepost. this.push(JSON.stringify(fieldRefined)); + lastInstanceId = instanceId; + if (isSubTable) lastRepeatId = field.__id; + added += 1; } + + // Controls the rows to be skipped based on skipToken + // Once set to true remains true + cursorPredicate = cursorPredicate || (skipToken.instanceId === instanceId && skipToken.repeatId === field.__id); + counted += 1; } done(); // signifies that this stream element is fully processed. @@ -406,12 +432,17 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t flush(done) { // flush is called just before the transform stream is done and closed; we write // our footer information, close the object, and tell the stream we are done. - this.push((counted <= doOffset) ? '{"value":[],' : '],'); // open object or close row array. + this.push((added === 0) ? '{"value":[],' : '],'); // open object or close row array. // if we were given an explicit count, use it from here out, to create // @odata.count and nextUrl. + const remaining = (tableRemaining != null) ? tableRemaining - added : remainingItems; const totalCount = (tableCount != null) ? tableCount : counted; - const nextUrl = nextUrlFor(limit, offset, totalCount, originalUrl); + + const skipTokenData = { instanceId: lastInstanceId }; + if (isSubTable) skipTokenData.repeatId = lastRepeatId; + + const nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); // we do toString on the totalCount because mustache won't bother rendering // the block if it sees integer zero. @@ -448,8 +479,10 @@ const singleRowToOData = (fields, row, domain, originalUrl, query) => { const table = tableParts.join('.'); if (!verifyTablePath(tableParts, fields)) throw Problem.user.notFound(); + const isSubTable = table !== 'Submissions'; + // extract all our fields first, the field extractor doesn't know about target contexts. - return submissionToOData(fields, table, row, options).then((subrows) => { + return submissionToOData(fields, table, row, options).then(({ data: subrows, instanceId }) => { // now we actually filter to the requested set. we actually only need to compare // the very last specified id, since it is fully unique. const filterContextIdx = targetContext.reduce(((extant, pair, idx) => ((pair[1] != null) ? idx : extant)), -1); @@ -462,12 +495,36 @@ const singleRowToOData = (fields, row, domain, originalUrl, query) => { const count = filtered.length; // now we can process $top/$skip/$count: - const { limit, offset, shouldCount } = extractPaging(table, query); - const nextUrl = nextUrlFor(limit, offset, count, originalUrl); + const paging = extractPaging(table, query); + const { limit, shouldCount, skipToken } = paging; + let { offset } = paging; + + if (skipToken) { + offset = filtered.fiindIndex(s => skipToken.repeatId === s.__id); + } + const pared = filtered.slice(offset, offset + limit); + let nextUrl = null; + + if (pared.length > 0) { + const remaining = count - (offset + limit); + + const skipTokenData = { + instanceId + }; + + if (isSubTable) skipTokenData.repeatId = pared[pared.length - 1].__id; + + nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); + } + + + // if $select is there and parentId is not requested then remove it + let paredRefined = options.metadata && !options.metadata[filterField] ? pared.map(p => without([filterField], p)) : pared; + // if $select is there and parentId is not requested then remove it - const paredRefined = options.metadata && !options.metadata[filterField] ? pared.map(p => without([filterField], p)) : pared; + paredRefined = options.metadata && !options.metadata.__id ? paredRefined.map(p => without(['__id'], p)) : paredRefined; // and finally splice together and return our result: const dataContents = paredRefined.map(JSON.stringify).join(','); diff --git a/lib/http/endpoint.js b/lib/http/endpoint.js index c8565abb4..a3a0060b7 100644 --- a/lib/http/endpoint.js +++ b/lib/http/endpoint.js @@ -288,7 +288,7 @@ const isJsonType = (x) => /(^|,)(application\/json|json)($|;|,)/i.test(x); const isXmlType = (x) => /(^|,)(application\/(atom(svc)?\+)?xml|atom|xml)($|;|,)/i.test(x); // various supported odata constants: -const supportedParams = [ '$format', '$count', '$skip', '$top', '$filter', '$wkt', '$expand', '$select' ]; +const supportedParams = [ '$format', '$count', '$skip', '$top', '$filter', '$wkt', '$expand', '$select', '$skiptoken' ]; const supportedFormats = { json: [ 'application/json', 'json' ], xml: [ 'application/xml', 'atom' ] diff --git a/lib/model/query/submissions.js b/lib/model/query/submissions.js index a11f1c5b8..3faeba804 100644 --- a/lib/model/query/submissions.js +++ b/lib/model/query/submissions.js @@ -199,9 +199,21 @@ const getById = (submissionId) => ({ maybeOne }) => maybeOne(sql`select * from submissions where id=${submissionId} and "deletedAt" is null`) .then(map(construct(Submission))); -const countByFormId = (formId, draft, options = QueryOptions.none) => ({ oneFirst }) => oneFirst(sql` -select count(*) from submissions - where ${equals({ formId, draft })} and "deletedAt" is null and ${odataFilter(options.filter, odataToColumnMap)}`); +const countByFormId = (formId, draft, options = QueryOptions.none) => ({ one }) => one(sql` +SELECT * FROM +( SELECT COUNT(*) count FROM submissions + WHERE ${equals({ formId, draft })} AND "deletedAt" IS NULL AND ${odataFilter(options.filter, odataToColumnMap)}) AS "all" +CROSS JOIN +( SELECT COUNT(*) remaining FROM submissions + ${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM submissions WHERE "instanceId" = ${options.skiptoken.instanceId}) AS cursor + ON submissions."createdAt" <= cursor."createdAt" AND submissions.id < cursor.id + `: sql``} + WHERE ${equals({ formId, draft })} + AND "deletedAt" IS NULL + AND ${odataFilter(options.filter, odataToColumnMap)}) AS skiptoken +`); const verifyVersion = (formId, rootId, instanceId, draft) => ({ maybeOne }) => maybeOne(sql` select true from submissions @@ -337,6 +349,11 @@ inner join (select "submissionId", (count(id) - 1) as count from submission_defs group by "submissionId") as edits on edits."submissionId"=submission_defs."submissionId" +${options.skiptoken && !options.skiptoken.repeatId ? sql` -- in case of subtable we are fetching all Submissions without pagination + inner join + (select id, "createdAt" from submissions where "instanceId" = ${options.skiptoken.instanceId}) as cursor + on submissions."createdAt" <= cursor."createdAt" and submissions.id < cursor.id +`: sql``} where ${encrypted ? sql`(form_defs."encKeyId" is null or form_defs."encKeyId" in (${sql.join(keyIds, sql`,`)})) and` : sql``} ${odataFilter(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)} and diff --git a/lib/resources/odata.js b/lib/resources/odata.js index b0560c7d0..efdf3b1fc 100644 --- a/lib/resources/odata.js +++ b/lib/resources/odata.js @@ -67,10 +67,10 @@ module.exports = (service, endpoint) => { Forms.getFields(form.def.id).then(selectFields(query, params.table)), Submissions.streamForExport(form.id, draft, undefined, options), ((params.table === 'Submissions') && options.hasPaging()) - ? Submissions.countByFormId(form.id, draft, options) : resolve(null) + ? Submissions.countByFormId(form.id, draft, options) : resolve({}) ]) - .then(([fields, stream, count]) => - json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count))); + .then(([fields, stream, { count, remaining }]) => + json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count, remaining))); }))); }; diff --git a/lib/util/db.js b/lib/util/db.js index f74d6adc3..962ddeb13 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -15,7 +15,7 @@ const { reject } = require('./promise'); const Problem = require('./problem'); const Option = require('./option'); const { PartialPipe, mapStream } = require('./stream'); -const { construct } = require('./util'); +const { construct, base64ToUtf8, utf8ToBase64 } = require('./util'); const { isTrue, isFalse } = require('./http'); const { Transform } = require('stream'); @@ -222,11 +222,17 @@ const equals = (obj) => { const page = (options) => { const parts = []; - if (options.offset != null) parts.push(sql`offset ${options.offset}`); + if (options.offset != null && !options.skiptoken) parts.push(sql`offset ${options.offset}`); if (options.limit != null) parts.push(sql`limit ${options.limit}`); return parts.length ? sql.join(parts, sql` `) : nothing; }; +const greaterThan = (k, v) => { + if (!k || !v) return sql`true`; + + return sql`${sql.identifier(k.split('.'))} > ${v}`; +}; + //////////////////////////////////////// // query func decorator // @@ -360,15 +366,27 @@ class QueryOptions { return f(this.args[arg]); } + static parseSkiptoken(token) { + const jsonString = base64ToUtf8(token); + return JSON.parse(jsonString); + } + + static getSkiptoken(data) { + const jsonString = JSON.stringify(data); + return utf8ToBase64(jsonString); + } + static fromODataRequest(params, query) { const result = { extended: true }; result.isSubmissionsTable = params.table === 'Submissions'; - if ((params.table === 'Submissions') && (query.$skip != null)) + if ((params.table === 'Submissions') && (!query.$skiptoken) && (query.$skip != null)) result.offset = parseInt(query.$skip, 10); if ((params.table === 'Submissions') && (query.$top != null)) result.limit = parseInt(query.$top, 10); if (query.$filter != null) result.filter = query.$filter; + if ((params.table === 'Submissions') && (query.$skiptoken != null)) + result.skiptoken = QueryOptions.parseSkiptoken(query.$skiptoken); return new QueryOptions(result); } @@ -532,7 +550,7 @@ const postgresErrorToProblem = (x) => { module.exports = { connectionString, connectionObject, - unjoiner, extender, equals, page, queryFuncs, + unjoiner, extender, equals, greaterThan, page, queryFuncs, insert, insertMany, updater, markDeleted, markUndeleted, QueryOptions, postgresErrorToProblem diff --git a/lib/util/odata.js b/lib/util/odata.js index cc2c93226..b28773109 100644 --- a/lib/util/odata.js +++ b/lib/util/odata.js @@ -13,6 +13,7 @@ const { max } = Math; const Problem = require('./problem'); const { parse, render } = require('mustache'); const { isTrue, urlWithQueryParams } = require('./http'); +const { QueryOptions } = require('./db'); const template = (body) => { parse(body); // caches template for future perf. @@ -49,10 +50,10 @@ const getServiceRoot = (subpath) => { // Given limit: Int, offset: Int, count: Int, originalUrl: String, calculates // what the nextUrl should be to supply server-driven paging (11.2.5.7). Returns // url: String? -const nextUrlFor = (limit, offset, count, originalUrl) => - ((offset + limit >= count) - ? null - : urlWithQueryParams(originalUrl, { $skip: (offset + limit), $top: null })); +const nextUrlFor = (remaining, originalUrl, skipTokenData) => ((!skipTokenData || remaining <= 0) + ? null + : urlWithQueryParams(originalUrl, { $skip: null, $skiptoken: QueryOptions.getSkiptoken(skipTokenData) })); + // Given a querystring object, returns an object of relevant OData options. Right // now that is only { wkt: Bool, expand: String, metadata: Array } diff --git a/lib/util/util.js b/lib/util/util.js index 0bb2de50a..cb3fa893c 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -60,6 +60,19 @@ const pickAll = (keys, obj) => { return result; }; +// source: https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem + +function base64ToUtf8(base64) { + const binString = atob(base64); + const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0)); + return new TextDecoder().decode(bytes); +} + +function utf8ToBase64(string) { + const bytes = new TextEncoder().encode(string); + const binString = Array.from(bytes, (x) => String.fromCodePoint(x)).join(''); + return btoa(binString); +} //////////////////////////////////////// // CLASSES @@ -76,6 +89,7 @@ module.exports = { noop, noargs, isBlank, isPresent, blankStringToNull, sanitizeOdataIdentifier, printPairs, without, pickAll, + base64ToUtf8, utf8ToBase64, construct }; diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index 49c24e225..5825b0825 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -126,7 +126,7 @@ describe('api: /datasets/:name.svc', () => { await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1') .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.be.equal('http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24skip=1'); + body['@odata.nextLink'].should.be.equal('http://localhost:8989/0?%24skiptoken=Mg%3D%3D'); }); })); diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 3b0692a67..c7d03ccc2 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -2,6 +2,8 @@ const { testService } = require('../setup'); const { sql } = require('slonik'); const testData = require('../../data/xml'); const { dissocPath, identity } = require('ramda'); +const { QueryOptions } = require('../../../lib/util/db'); +const should = require('should'); // NOTE: for the data output tests, we do not attempt to extensively determine if every // internal case is covered; there are already two layers of tests below these, at @@ -305,7 +307,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNmU5M2E4MWE1M2VlZDA1NjZlNjVlNDcyZDRhNGI5YWUzODNlZTZkIn0%3D', value: [{ __id: 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', '__Submissions-id': 'double', @@ -320,11 +322,10 @@ describe('api: /forms/:id.svc', () => { it('should return just a count if asked', testService((service) => withSubmission(service, (asAlice) => asAlice.get("/v1/projects/1/forms/doubleRepeat.svc/Submissions('double')/children/child?$top=0&$count=true") - .expect(200) + // .expect(200) .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24count=true&%24skip=0', '@odata.count': 3, value: [] }); @@ -387,7 +388,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get("/v1/projects/1/forms/double%20repeat.svc/Submissions('uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4')/children/child?$top=1") .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24skip=1'); + body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidXVpZDoxN2IwOWU5Ni00MTQxLTQzZjUtOWE3MC02MTFlYjBlOGY2YjQiLCJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); }) ])))))); @@ -691,7 +692,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byJ9', value: [{ __id: 'rtwo', __system: { @@ -720,64 +721,77 @@ describe('api: /forms/:id.svc', () => { // nb: order of id and createdAt is not guaranteed to be same // in test env, see submission id 134849 and 134850 // 50 (at 873 ms) was created before 49 (at 874 ms) - it.skip('should limit Submissions', testService(async (service) => { + it('should limit Submissions', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10') - .expect(200) + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1') + // .expect(200) .then(({ body }) => { const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=${(token)}`); }); })); - it.skip('should ignore $skip when $skipToken is given', testService(async (service) => { + it('should ignore $skip when $skipToken is given', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skip=10') + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$skip=1') .expect(200) .then(({ body }) => { - body.value[0].instanceName.should.be.eql('eleven'); - body.value[9].instanceName.should.be.eql('twenty'); + body.value[0].__id.should.be.eql('rtwo'); const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + + const expectedNextLink = `http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=${(token)}`; + body['@odata.nextLink'].should.eql(expectedNextLink); return body['@odata.nextLink']; }); - await asAlice.get(nextlink + '&$skip=10') + await asAlice.get(nextlink.replace('http://localhost:8989', '') + '&$skip=1') .expect(200) .then(({ body }) => { - body.value[0].instanceName.should.be.eql('twenty one'); - body.value[9].instanceName.should.be.eql('thirty'); + body.value[0].__id.should.be.eql('rone'); - const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$skipToken=${token}`); + should.not.exist(body['@odata.nextLink']); }); })); - it.skip('should limit and filter Submissions', testService(async (service) => { + it('should limit and filter Submissions', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$filter=__system/reviewState eq 'rejected'`) - .expect(200) + await asAlice.patch('/v1/projects/1/forms/withrepeat/submissions/rtwo') + .send({ reviewState: 'rejected' }) + .expect(200); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$filter=not __system/reviewState eq \'rejected\'') + // .expect(200) .then(({ body }) => { const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?$top=10&$filter=__system/reviewState eq 'rejected'&$skipToken=${token}`); + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24filter=not+__system%2FreviewState+eq+%27rejected%27&%24skiptoken=${token}`); + }); + })); + + it('should limit Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$select=age') + // .expect(200) + .then(({ body }) => { + body.value[0].should.be.eql({ + age: 38, + }); + body['@odata.nextLink'].should.be.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24select=age&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D'); }); })); @@ -792,7 +806,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24count=true&%24skip=1', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24count=true&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D', '@odata.count': 3, value: [{ __id: 'rthree', @@ -1349,7 +1363,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9', value: [{ __id: '52eff9ea82550183880b9d64c20487642fa6e60c', '__Submissions-id': 'rtwo', @@ -1430,37 +1444,57 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body['@odata.count'].should.equal(2); - }) + }); })); - it.skip('should limit subtable results', testService(async (service) => { + it('should limit subtable results', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10') + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=1') .expect(200) .then(({ body }) => { - const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - __id: body.value[body.value.length - 1].__id, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$skipToken=${token}`); + body.value[0].name.should.be.eql('Candace'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIiwicmVwZWF0SWQiOiIzMjgwOWFlMmIzZGM0MDRlYTI5MjIwNWViODg0YjIxZmE0ZTlhY2M1In0%3D'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + + body.value[0].name.should.be.eql('Billy'); + + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + // should.not.exist(body['@odata.nextLink']); }); })); - it.skip('should limit and filter subtable', testService(async (service) => { + it('should limit and filter subtable', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - await asAlice.get(`/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$filter=$root/Submission/__system/reviewState eq 'rejected'`) + await asAlice.patch('/v1/projects/1/forms/withrepeat/submissions/rtwo') + .send({ reviewState: 'rejected' }) + .expect(200); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=1&$filter=$root/Submissions/__system/reviewState eq \'rejected\'') .expect(200) .then(({ body }) => { - const tokenData = { - instanceId: body.value[body.value.length - 1].instanceId, - __id: body.value[body.value.length - 1].__id, - } - const token = btoa(JSON.stringify(tokenData)); - body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=10&$filter=$root/Submission/__system/reviewState eq 'rejected'&$skipToken=${token}`); + body.value[0].name.should.be.eql('Billy'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + + body.value[0].name.should.be.eql('Blaine'); + + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + should.not.exist(body['@odata.nextLink']); }); + + })); // we cheat here. see mark1. diff --git a/test/unit/data/odata.js b/test/unit/data/odata.js index 5286ae0c9..4f5f25d15 100644 --- a/test/unit/data/odata.js +++ b/test/unit/data/odata.js @@ -39,7 +39,7 @@ describe('submissionToOData', () => { it('should parse and transform a basic submission', () => fieldsFor(testData.forms.simple).then((fields) => { const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'one', __system, @@ -60,7 +60,7 @@ describe('submissionToOData', () => { // have one for explicity this purpose in case things change. it('should include submission metadata on the root output', () => { const submission = mockSubmission('test', testData.instances.simple.one); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system }]); }); }); @@ -68,7 +68,7 @@ describe('submissionToOData', () => { it('should set the correct review state', () => { const submission = Object.assign(mockSubmission('test', testData.instances.simple.one), { reviewState: 'hasIssues' }); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { reviewState: 'hasIssues' }) }]); }); }); @@ -76,7 +76,7 @@ describe('submissionToOData', () => { it('should set the correct deviceId', () => { const submission = Object.assign(mockSubmission('test', testData.instances.simple.one), { deviceId: 'cool device' }); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { deviceId: 'cool device' }) }]); }); }); @@ -85,7 +85,7 @@ describe('submissionToOData', () => { const submission = mockSubmission('test', testData.instances.simple.one); submission.aux.edit = { count: 42 }; - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { edits: 42 }) }]); }); }); @@ -93,7 +93,7 @@ describe('submissionToOData', () => { it('should not crash if no submitter exists', () => { const submission = mockSubmission('test', testData.instances.simple.one); submission.aux.submitter = {}; // wipe it back out. - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: { @@ -138,7 +138,7 @@ describe('submissionToOData', () => { hello what could it be? `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'types', __system, @@ -164,7 +164,7 @@ describe('submissionToOData', () => { new MockField({ path: '/uranus', name: 'uranus', type: 'repeat' }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -188,7 +188,7 @@ describe('submissionToOData', () => { new MockField({ path: '/neptune', name: 'neptune', type: 'structure', order: 6 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -211,7 +211,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 5 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -235,7 +235,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 5 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions.sun', submission).then((result) => { + return submissionToOData(fields, 'Submissions.sun', submission).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'nulls', __id: '68874cc5985b68898fbd0af1156e12b6270820f7', @@ -258,7 +258,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 6 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions.sun', submission).then((result) => { + return submissionToOData(fields, 'Submissions.sun', submission).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'nulls', __id: '68874cc5985b68898fbd0af1156e12b6270820f7', @@ -279,7 +279,7 @@ describe('submissionToOData', () => { hello <42>108 `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'sanitize', __system, @@ -301,7 +301,7 @@ describe('submissionToOData', () => { dos `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'sanitize2', __system, @@ -315,7 +315,7 @@ describe('submissionToOData', () => { const submission = mockSubmission('entities', ` «hello» `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'entities', __system, @@ -333,7 +333,7 @@ describe('submissionToOData', () => { 100 this is nonsensical `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geo', __system, @@ -352,7 +352,7 @@ describe('submissionToOData', () => { 4.8 15.16 23.42 11.38 -11.38 `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -371,7 +371,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8; 11.1 22.2;33.3 44.4;55.5 66.6; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -399,7 +399,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8; 11.1 22.2;33.3 44.4;55.5 66.6; `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -418,7 +418,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8;10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2;33.3 44.4;55.5 66.6;11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -446,7 +446,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4; 5.5 6.6 7.7 8.8; 10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2; 33.3 44.4;55.5 66.6; 11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -474,7 +474,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8;10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2;33.3 44.4;55.5 66.6;11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -492,7 +492,7 @@ describe('submissionToOData', () => { new MockField({ path: '/age', name: 'age', type: 'int' }) ]; const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'one', __system, @@ -518,7 +518,7 @@ describe('submissionToOData', () => { tres `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nesting', __system, @@ -530,7 +530,7 @@ describe('submissionToOData', () => { it('should provide navigation links for repeats', () => fieldsFor(testData.forms.withrepeat).then((fields) => { const submission = mockSubmission('rtwo', testData.instances.withrepeat.two); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'rtwo', __system, @@ -549,7 +549,7 @@ describe('submissionToOData', () => { .then((fields) => { const submission = mockSubmission('two', testData.instances.doubleRepeat.double); return submissionToOData(fields, 'Submissions', submission, { expand: '*' }) - .then((result) => { + .then(({ data: result }) => { result.should.eql([ { __id: 'two', @@ -635,7 +635,7 @@ describe('submissionToOData', () => { it('should extract subtable rows within repeats', () => fieldsFor(testData.forms.withrepeat).then((fields) => { const row = { instanceId: 'two', xml: testData.instances.withrepeat.two, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', row).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'two', __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', @@ -653,7 +653,7 @@ describe('submissionToOData', () => { it('should return navigation links to repeats within a subtable result set', () => fieldsFor(testData.forms.doubleRepeat).then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', row).then(({ data: result }) => { result.should.eql([{ __id: '46ebf42ee83ddec5028c42b2c054402d1e700208', '__Submissions-id': 'double', @@ -680,7 +680,7 @@ describe('submissionToOData', () => { it('should return second-order subtable results', () => fieldsFor(testData.forms.doubleRepeat).then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child.toys.toy', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child.toys.toy', row).then(({ data: result }) => { result.should.eql([{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -724,7 +724,7 @@ describe('submissionToOData', () => { new MockField({ path: '/name', name: 'name', type: 'string' }) ]; const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission, { metadata: { __id: true, '__system/status': true } }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { metadata: { __id: true, '__system/status': true } }).then(({ data: result }) => { result.should.eql([{ __id: 'one', __system: { status: null }, @@ -740,7 +740,7 @@ describe('submissionToOData', () => { new MockField({ path: '/children/child', name: 'child', type: 'repeat' }) ]; const submission = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', submission, { metadata: { __id: true } }).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', submission, { metadata: { __id: true } }).then(({ data: result }) => { result.should.eql([{ __id: '46ebf42ee83ddec5028c42b2c054402d1e700208', '__Submissions-id': 'double' @@ -759,7 +759,7 @@ describe('submissionToOData', () => { .then(selectFields({ $select: 'name' }, 'Submissions.children.child.toys.toy')) .then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child.toys.toy', row, { metadata: { __id: true } }).then((result) => { + return submissionToOData(fields, 'Submissions.children.child.toys.toy', row, { metadata: { __id: true } }).then(({ data: result }) => { result.should.eql([{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', diff --git a/test/unit/formats/odata.js b/test/unit/formats/odata.js index 47991c637..1a4bdeead 100644 --- a/test/unit/formats/odata.js +++ b/test/unit/formats/odata.js @@ -577,10 +577,10 @@ describe('odata message composition', () => { const query = { $top: '3', $skip: '2' }; const inRows = streamTest.fromObjects(instances(6)); // make it close to check the off-by-one. fieldsFor(testData.forms.simple) - .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2', query, inRows)) + .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24skip=5'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24skiptoken=e30%3D'); done(); }))); }); @@ -589,10 +589,10 @@ describe('odata message composition', () => { const query = { $top: '3', $skip: '2', $wkt: 'true', $count: 'true' }; const inRows = streamTest.fromObjects(instances(6)); // make it close to check the off-by-one. fieldsFor(testData.forms.simple) - .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2&$wkt=true&$count=true', query, inRows)) + .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2&$wkt=true&$count=true', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24skip=5&%24wkt=true&%24count=true'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24wkt=true&%24count=true&%24skiptoken=e30%3D'); done(); }))); }); @@ -669,7 +669,6 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/simple.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/simple.svc/Submissions?%24skip=2', value: [ { __id: 'one', __system, meta: { instanceID: 'one' }, name: 'Alice', age: 30 }, { __id: 'two', __system, meta: { instanceID: 'two' }, name: 'Bob', age: 34 }, @@ -746,7 +745,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', value: [{ __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', '__Submissions-id': 'two', @@ -803,7 +802,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', value: [{ __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', '__Submissions-id': 'two', @@ -900,7 +899,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24skip=1"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); }); }); }); @@ -913,7 +912,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1&$wkt=true", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24wkt=true&%24skip=1"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); }); }); }); @@ -1010,7 +1009,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24skip=2", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiI4ZDJkYzdiZDNlOTdhNjkwYzA4MTNlNjQ2NjU4ZTUxMDM4ZWI0MTQ0In0%3D", value: [{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -1063,7 +1062,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24skip=3", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNzE2ZGQ4Yjc5YTRjOTM2OWQ2YjFlN2E5YzlkNTVhYzE4ZGExMzE5In0%3D", value: [{ __id: '8d2dc7bd3e97a690c0813e646658e51038eb4144', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', diff --git a/test/unit/util/util.js b/test/unit/util/util.js index 6408f02f1..bd7e44749 100644 --- a/test/unit/util/util.js +++ b/test/unit/util/util.js @@ -63,5 +63,13 @@ describe('util/util', () => { }); }); + describe('UTF-8 and Base64 conversion', () => { + const { utf8ToBase64, base64ToUtf8 } = util; + it('should convert unicode to base64', () => { + const input = 'a Ā 𐀀 文 🦄'; + const base64 = utf8ToBase64(input); + base64ToUtf8(base64).should.be.eql(input); + }); + }); }); From 44a2714e1e90794ac7ad4fbce6843f965b032b36 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Mon, 31 Jul 2023 11:25:09 -0400 Subject: [PATCH 03/10] changes for Entiites Odata --- lib/data/entity.js | 10 +++- lib/model/query/entities.js | 27 ++++++++- lib/resources/odata-entities.js | 4 +- lib/util/db.js | 4 +- lib/util/odata.js | 5 +- test/integration/api/odata-entities.js | 77 +++++++++++++++++++++++++- 6 files changed, 113 insertions(+), 14 deletions(-) diff --git a/lib/data/entity.js b/lib/data/entity.js index 411982179..9478de8bd 100644 --- a/lib/data/entity.js +++ b/lib/data/entity.js @@ -263,12 +263,13 @@ const extractSelectedProperties = (query, properties) => { }; // Pagination is done at the database level -const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tableCount) => { +const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tableCount, tableRemaining) => { const serviceRoot = getServiceRoot(originalUrl); - const { limit, offset, shouldCount } = extractPaging(query); + const { limit, offset, shouldCount, skipToken } = extractPaging(query); const selectedProperties = extractSelectedProperties(query, properties); let isFirstEntity = true; + let lastUuid; const rootStream = new Transform({ writableObjectMode: true, // we take a stream of objects from the db, but readableObjectMode: false, // we put out a stream of text. @@ -281,6 +282,8 @@ const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tab this.push(','); } + lastUuid = entity.uuid; + this.push(JSON.stringify(selectFields(entity, properties, selectedProperties))); done(); @@ -290,8 +293,9 @@ const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tab }, flush(done) { this.push((isFirstEntity) ? '{"value":[],' : '],'); // open object or close row array. + const remaining = skipToken ? tableRemaining - limit : tableCount - (limit + offset); // @odata.count and nextUrl. - const nextUrl = nextUrlFor(limit, offset, tableCount, originalUrl); + const nextUrl = nextUrlFor(remaining, originalUrl, { uuid: lastUuid }); this.push(jsonDataFooter({ table: 'Entities', domain, serviceRoot, nextUrl, count: (shouldCount ? tableCount.toString() : null) })); done(); diff --git a/lib/model/query/entities.js b/lib/model/query/entities.js index d16250d11..7c5d1ecb3 100644 --- a/lib/model/query/entities.js +++ b/lib/model/query/entities.js @@ -315,6 +315,11 @@ INNER JOIN SELECT "entityId", (COUNT(id) - 1) AS "updates" FROM entity_defs GROUP BY "entityId" ) stats ON stats."entityId"=entity_defs."entityId" LEFT JOIN actors ON entities."creatorId"=actors.id +${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM entities WHERE "uuid" = ${options.skiptoken.uuid}) AS cursor + ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id + `: sql``} WHERE entities."datasetId" = ${datasetId} AND entities."deletedAt" IS NULL @@ -324,10 +329,26 @@ ORDER BY entities."createdAt" DESC, entities.id DESC ${page(options)}`) .then(stream.map(_exportUnjoiner)); -const countByDatasetId = (datasetId, options = QueryOptions.none) => ({ oneFirst }) => oneFirst(sql` -SELECT count(*) FROM entities +const countByDatasetId = (datasetId, options = QueryOptions.none) => ({ one }) => one(sql` +SELECT * FROM + +( + SELECT count(*) count FROM entities + WHERE "datasetId" = ${datasetId} + AND ${odataFilter(options.filter, odataToColumnMap)} +) AS "all" + +CROSS JOIN +( + SELECT COUNT(*) remaining FROM entities + ${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM entities WHERE "uuid" = ${options.skiptoken.uuid}) AS cursor + ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id + `: sql``} WHERE "datasetId" = ${datasetId} - AND ${odataFilter(options.filter, odataToColumnMap)}`); + AND ${odataFilter(options.filter, odataToColumnMap)} +) AS skiptoken`); //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/resources/odata-entities.js b/lib/resources/odata-entities.js index 74cf54402..1859cd18d 100644 --- a/lib/resources/odata-entities.js +++ b/lib/resources/odata-entities.js @@ -46,9 +46,9 @@ module.exports = (service, endpoint) => { const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound); const properties = await Datasets.getProperties(dataset.id); const options = QueryOptions.fromODataRequestEntities(query); - const count = await Entities.countByDatasetId(dataset.id, options); + const { count, remaining } = await Entities.countByDatasetId(dataset.id, options); const entities = await Entities.streamForExport(dataset.id, options); - return json(streamEntityOdata(entities, properties, env.domain, originalUrl, query, count)); + return json(streamEntityOdata(entities, properties, env.domain, originalUrl, query, count, remaining)); })); diff --git a/lib/util/db.js b/lib/util/db.js index 962ddeb13..2c8c51a00 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -393,12 +393,14 @@ class QueryOptions { static fromODataRequestEntities(query) { const result = { extended: true }; - if (query.$skip != null) + if (!query.$skiptoken && query.$skip != null) result.offset = parseInt(query.$skip, 10); if (query.$top != null) result.limit = parseInt(query.$top, 10); if (query.$filter != null) result.filter = query.$filter; + if (query.$skiptoken != null) + result.skiptoken = QueryOptions.parseSkiptoken(query.$skiptoken); return new QueryOptions(result); } diff --git a/lib/util/odata.js b/lib/util/odata.js index b28773109..417c4abef 100644 --- a/lib/util/odata.js +++ b/lib/util/odata.js @@ -23,9 +23,10 @@ const template = (body) => { const extractPaging = (query) => { const parsedLimit = parseInt(query.$top, 10); const limit = Number.isNaN(parsedLimit) ? Infinity : parsedLimit; - const offset = parseInt(query.$skip, 10) || 0; + const offset = (!query.$skiptoken && parseInt(query.$skip, 10)) || 0; const shouldCount = isTrue(query.$count); - const result = { limit: max(0, limit), offset: max(0, offset), shouldCount }; + const skipToken = query.$skiptoken ? QueryOptions.parseSkiptoken(query.$skiptoken) : null; + const result = { limit: max(0, limit), offset: max(0, offset), shouldCount, skipToken }; return Object.assign(result, { doLimit: Infinity, doOffset: 0 }); }; diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index 5825b0825..0ef6a1064 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -12,6 +12,8 @@ const testData = require('../../data/xml'); const { exhaust } = require('../../../lib/worker/worker'); const { v4: uuid } = require('uuid'); const { sql } = require('slonik'); +const { QueryOptions } = require('../../../lib/util/db'); +const should = require('should'); describe('api: /datasets/:name.svc', () => { describe('GET /Entities', () => { @@ -21,6 +23,7 @@ describe('api: /datasets/:name.svc', () => { await user.post('/v1/projects/1/forms/simpleEntity/submissions') .send(testData.instances.simpleEntity.one .replace(/one/g, `submission${i+skip}`) + .replace(/88/g, i + skip + 1) .replace('uuid:12345678-1234-4123-8234-123456789abc', uuid())) .set('Content-Type', 'application/xml') .expect(200); @@ -49,9 +52,9 @@ describe('api: /datasets/:name.svc', () => { .then(({ body }) => { body.value.length.should.be.eql(2); - body.value.forEach(r => { + body.value.forEach((r, i) => { r.first_name.should.be.eql('Alice'); - r.age.should.be.eql('88'); + r.age.should.be.eql((2 - i).toString()); }); body.value[0].__id.should.not.be.eql(body.value[1].__id); @@ -126,10 +129,49 @@ describe('api: /datasets/:name.svc', () => { await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1') .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.be.equal('http://localhost:8989/0?%24skiptoken=Mg%3D%3D'); + const tokenData = { + uuid: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24skiptoken=${token}`); }); })); + it('should not duplicate or skip entities - opaque cursor', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 2); + + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1') + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('2'); + const tokenData = { + uuid: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24skiptoken=${token}`); + return body['@odata.nextLink']; + }); + + // create of these 2 entities have no impact on the nextlink + await createSubmissions(asAlice, container, 2, 2); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('1'); + should.not.exist(body['@odata.nextLink']); + }); + + + })); + it('should return filtered entities', testService(async (service, container) => { const asAlice = await service.login('alice'); @@ -151,6 +193,35 @@ describe('api: /datasets/:name.svc', () => { }); })); + it('should return filtered entities with pagination', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 2); + + await container.run(sql`UPDATE entities SET "createdAt" = '2020-01-01'`); + + await createSubmissions(asAlice, container, 2, 2); + + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1&$filter=__system/createdAt gt 2021-01-01') + .expect(200) + .then(({ body }) => { + body.value.length.should.be.eql(1); + body.value[0].age.should.be.eql('4'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('3'); + }); + })); + it('should throw error if filter criterion is invalid', testService(async (service, container) => { const asAlice = await service.login('alice'); From 8cad0da57e4b8b70a67ef5749f6b7aeda35f6db1 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Mon, 31 Jul 2023 13:07:13 -0400 Subject: [PATCH 04/10] refactored a bit and added unit tests --- lib/data/odata.js | 4 ---- lib/formats/odata.js | 14 +++++++++++--- lib/util/db.js | 8 +------- test/unit/data/odata.js | 3 ++- test/unit/formats/odata.js | 29 +++++++++++++++++++++++++++++ test/unit/util/db.js | 9 +++++++++ 6 files changed, 52 insertions(+), 15 deletions(-) diff --git a/lib/data/odata.js b/lib/data/odata.js index 038fd1b12..e7f0906bd 100644 --- a/lib/data/odata.js +++ b/lib/data/odata.js @@ -136,9 +136,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // on option.metadata even though they are not part of the form's own schema. // So rather than try to inject them into the xml transformation below, we just // formulate them herein advance: - // if (!options.metadata || options.metadata.__id) { root.__id = submission.instanceId; - // } if (table === 'Submissions') { const systemObj = { submissionDate: submission.createdAt, @@ -220,9 +218,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // create our new databag, push into result data, and set it as our result ptr. const bag = generateDataFrame(schemaStack); - // if (!options.metadata || options.metadata.__id) { bag.__id = hashId(schemaStack, submission.instanceId); - // } dataPtr[outname].push(bag); dataStack.push(bag); diff --git a/lib/formats/odata.js b/lib/formats/odata.js index 1474052d8..6b8857e50 100644 --- a/lib/formats/odata.js +++ b/lib/formats/odata.js @@ -382,12 +382,17 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t if (!verifyTablePath(table.split('.'), fields)) throw Problem.user.notFound(); // write the header, then transform and stream each row. + // To count total number of items for subtable (repeats) let counted = 0; - let lastInstanceId = null; - let lastRepeatId = null; + // To count items added to the downstream, required only for subtable let added = 0; + // To count remaining items in case of subtable let remainingItems = 0; + // skipToken is created based on following two variables + let lastInstanceId = null; + let lastRepeatId = null; + // For Submissions table, it is true because cursor is handled at database level let cursorPredicate = !isSubTable || !skipToken; @@ -436,9 +441,12 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t // if we were given an explicit count, use it from here out, to create // @odata.count and nextUrl. - const remaining = (tableRemaining != null) ? tableRemaining - added : remainingItems; const totalCount = (tableCount != null) ? tableCount : counted; + // How many items are remaining for the next page? + // if there aren't any then we don't need nextUrl + const remaining = (tableRemaining != null) ? tableRemaining - added : remainingItems; + const skipTokenData = { instanceId: lastInstanceId }; if (isSubTable) skipTokenData.repeatId = lastRepeatId; diff --git a/lib/util/db.js b/lib/util/db.js index 2c8c51a00..18be859ac 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -227,12 +227,6 @@ const page = (options) => { return parts.length ? sql.join(parts, sql` `) : nothing; }; -const greaterThan = (k, v) => { - if (!k || !v) return sql`true`; - - return sql`${sql.identifier(k.split('.'))} > ${v}`; -}; - //////////////////////////////////////// // query func decorator // @@ -552,7 +546,7 @@ const postgresErrorToProblem = (x) => { module.exports = { connectionString, connectionObject, - unjoiner, extender, equals, greaterThan, page, queryFuncs, + unjoiner, extender, equals, page, queryFuncs, insert, insertMany, updater, markDeleted, markUndeleted, QueryOptions, postgresErrorToProblem diff --git a/test/unit/data/odata.js b/test/unit/data/odata.js index 4f5f25d15..ea2483325 100644 --- a/test/unit/data/odata.js +++ b/test/unit/data/odata.js @@ -39,7 +39,8 @@ describe('submissionToOData', () => { it('should parse and transform a basic submission', () => fieldsFor(testData.forms.simple).then((fields) => { const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result, instanceId }) => { + instanceId.should.be.eql('one'); result.should.eql([{ __id: 'one', __system, diff --git a/test/unit/formats/odata.js b/test/unit/formats/odata.js index 1a4bdeead..b64c4a8de 100644 --- a/test/unit/formats/odata.js +++ b/test/unit/formats/odata.js @@ -8,6 +8,7 @@ const { fieldsFor, MockField } = require(appRoot + '/test/util/schema'); // eslint-disable-next-line import/no-dynamic-require const testData = require(appRoot + '/test/data/xml'); const should = require('should'); +const { QueryOptions } = require('../../../lib/util/db'); // Helpers to deal with repeated system metadata generation. const submitter = { id: 5, displayName: 'Alice' }; @@ -813,6 +814,34 @@ describe('odata message composition', () => { done(); }))); }); + + it('should offset subtable row data by skipToken', (done) => { + const query = { $skiptoken: QueryOptions.getSkiptoken({ instanceId: 'two', repeatId: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d' }) }; + const inRows = streamTest.fromObjects([ + mockSubmission('one', testData.instances.withrepeat.one), + mockSubmission('two', testData.instances.withrepeat.two), + mockSubmission('three', testData.instances.withrepeat.three) + ]); + fieldsFor(testData.forms.withrepeat) + .then((fields) => rowStreamToOData(fields, 'Submissions.children.child', 'http://localhost:8989', '/withrepeat.svc/Submissions.children.child?$skip=1&$top=1', query, inRows)) + .then((stream) => stream.pipe(streamTest.toText((_, result) => { + JSON.parse(result).should.eql({ + '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', + value: [{ + __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', + '__Submissions-id': 'two', + name: 'Blaine', + age: 6 + }, { + __id: 'beaedcdba519e6e6b8037605c9ae3f6a719984fa', + '__Submissions-id': 'three', + name: 'Candace', + age: 2 + }] + }); + done(); + }))); + }); }); }); diff --git a/test/unit/util/db.js b/test/unit/util/db.js index 8fdf43193..eaa7ba0a2 100644 --- a/test/unit/util/db.js +++ b/test/unit/util/db.js @@ -424,6 +424,15 @@ returning *`); .args.should.eql({ b: 2, c: 3, f: 9 }); }); + it('should create and parse cursor token', () => { + const data = { + someid: '123' + }; + + const token = QueryOptions.getSkiptoken(data); + QueryOptions.parseSkiptoken(token).should.be.eql(data); + }); + describe('related functions', () => { it('should run the handler only if the arg is present', () => { let ran = false; From b76c3887a7b8e62ceb22d6ec5f793a6a23928eed Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Thu, 3 Aug 2023 11:50:34 -0400 Subject: [PATCH 05/10] prefix skiptoken with 01 --- lib/util/db.js | 4 ++-- test/integration/api/odata.js | 20 ++++++++++---------- test/unit/formats/odata.js | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/util/db.js b/lib/util/db.js index 18be859ac..2df12907e 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -361,13 +361,13 @@ class QueryOptions { } static parseSkiptoken(token) { - const jsonString = base64ToUtf8(token); + const jsonString = base64ToUtf8(token.substr(2)); return JSON.parse(jsonString); } static getSkiptoken(data) { const jsonString = JSON.stringify(data); - return utf8ToBase64(jsonString); + return '01' + utf8ToBase64(jsonString); // 01 is the version number of this scheme } static fromODataRequest(params, query) { diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index c7d03ccc2..53df57aa1 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -307,7 +307,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNmU5M2E4MWE1M2VlZDA1NjZlNjVlNDcyZDRhNGI5YWUzODNlZTZkIn0%3D', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNmU5M2E4MWE1M2VlZDA1NjZlNjVlNDcyZDRhNGI5YWUzODNlZTZkIn0%3D', value: [{ __id: 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', '__Submissions-id': 'double', @@ -388,7 +388,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get("/v1/projects/1/forms/double%20repeat.svc/Submissions('uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4')/children/child?$top=1") .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidXVpZDoxN2IwOWU5Ni00MTQxLTQzZjUtOWE3MC02MTFlYjBlOGY2YjQiLCJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); + body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidXVpZDoxN2IwOWU5Ni00MTQxLTQzZjUtOWE3MC02MTFlYjBlOGY2YjQiLCJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); }) ])))))); @@ -692,7 +692,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byJ9', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byJ9', value: [{ __id: 'rtwo', __system: { @@ -791,7 +791,7 @@ describe('api: /forms/:id.svc', () => { body.value[0].should.be.eql({ age: 38, }); - body['@odata.nextLink'].should.be.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24select=age&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D'); + body['@odata.nextLink'].should.be.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24select=age&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D'); }); })); @@ -806,7 +806,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24count=true&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24count=true&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D', '@odata.count': 3, value: [{ __id: 'rthree', @@ -1363,7 +1363,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9', value: [{ __id: '52eff9ea82550183880b9d64c20487642fa6e60c', '__Submissions-id': 'rtwo', @@ -1454,7 +1454,7 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body.value[0].name.should.be.eql('Candace'); - body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnRocmVlIiwicmVwZWF0SWQiOiIzMjgwOWFlMmIzZGM0MDRlYTI5MjIwNWViODg0YjIxZmE0ZTlhY2M1In0%3D'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIiwicmVwZWF0SWQiOiIzMjgwOWFlMmIzZGM0MDRlYTI5MjIwNWViODg0YjIxZmE0ZTlhY2M1In0%3D'); return body['@odata.nextLink']; }); @@ -1464,7 +1464,7 @@ describe('api: /forms/:id.svc', () => { body.value[0].name.should.be.eql('Billy'); - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" // should.not.exist(body['@odata.nextLink']); }); })); @@ -1480,7 +1480,7 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body.value[0].name.should.be.eql('Billy'); - body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9'); return body['@odata.nextLink']; }); @@ -1490,7 +1490,7 @@ describe('api: /forms/:id.svc', () => { body.value[0].name.should.be.eql('Blaine'); - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" should.not.exist(body['@odata.nextLink']); }); diff --git a/test/unit/formats/odata.js b/test/unit/formats/odata.js index b64c4a8de..9e9409bb6 100644 --- a/test/unit/formats/odata.js +++ b/test/unit/formats/odata.js @@ -581,7 +581,7 @@ describe('odata message composition', () => { .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24skiptoken=e30%3D'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24skiptoken=01e30%3D'); done(); }))); }); @@ -593,7 +593,7 @@ describe('odata message composition', () => { .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2&$wkt=true&$count=true', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24wkt=true&%24count=true&%24skiptoken=e30%3D'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24wkt=true&%24count=true&%24skiptoken=01e30%3D'); done(); }))); }); @@ -746,7 +746,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', value: [{ __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', '__Submissions-id': 'two', @@ -803,7 +803,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', value: [{ __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', '__Submissions-id': 'two', @@ -928,7 +928,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); }); }); }); @@ -941,7 +941,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1&$wkt=true", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); }); }); }); @@ -1038,7 +1038,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiI4ZDJkYzdiZDNlOTdhNjkwYzA4MTNlNjQ2NjU4ZTUxMDM4ZWI0MTQ0In0%3D", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiI4ZDJkYzdiZDNlOTdhNjkwYzA4MTNlNjQ2NjU4ZTUxMDM4ZWI0MTQ0In0%3D", value: [{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -1091,7 +1091,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNzE2ZGQ4Yjc5YTRjOTM2OWQ2YjFlN2E5YzlkNTVhYzE4ZGExMzE5In0%3D", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNzE2ZGQ4Yjc5YTRjOTM2OWQ2YjFlN2E5YzlkNTVhYzE4ZGExMzE5In0%3D", value: [{ __id: '8d2dc7bd3e97a690c0813e646658e51038eb4144', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', From 9d5857574821dd78aaebcdf625c66ac952756c78 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Thu, 3 Aug 2023 12:22:20 -0400 Subject: [PATCH 06/10] just use repeatId for subtables --- lib/formats/odata.js | 10 +++++----- test/integration/api/odata.js | 14 +++++++------- test/unit/formats/odata.js | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/formats/odata.js b/lib/formats/odata.js index 6b8857e50..f71011a85 100644 --- a/lib/formats/odata.js +++ b/lib/formats/odata.js @@ -427,7 +427,7 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t // Controls the rows to be skipped based on skipToken // Once set to true remains true - cursorPredicate = cursorPredicate || (skipToken.instanceId === instanceId && skipToken.repeatId === field.__id); + cursorPredicate = cursorPredicate || skipToken.repeatId === field.__id; counted += 1; } @@ -447,8 +447,8 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t // if there aren't any then we don't need nextUrl const remaining = (tableRemaining != null) ? tableRemaining - added : remainingItems; - const skipTokenData = { instanceId: lastInstanceId }; - if (isSubTable) skipTokenData.repeatId = lastRepeatId; + let skipTokenData = { instanceId: lastInstanceId }; + if (isSubTable) skipTokenData = { repeatId: lastRepeatId }; const nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); @@ -518,11 +518,11 @@ const singleRowToOData = (fields, row, domain, originalUrl, query) => { if (pared.length > 0) { const remaining = count - (offset + limit); - const skipTokenData = { + let skipTokenData = { instanceId }; - if (isSubTable) skipTokenData.repeatId = pared[pared.length - 1].__id; + if (isSubTable) skipTokenData = { repeatId: pared[pared.length - 1].__id }; nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); } diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 53df57aa1..7eb88033d 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -307,7 +307,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNmU5M2E4MWE1M2VlZDA1NjZlNjVlNDcyZDRhNGI5YWUzODNlZTZkIn0%3D', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImI2ZTkzYTgxYTUzZWVkMDU2NmU2NWU0NzJkNGE0YjlhZTM4M2VlNmQifQ%3D%3D', value: [{ __id: 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', '__Submissions-id': 'double', @@ -388,7 +388,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get("/v1/projects/1/forms/double%20repeat.svc/Submissions('uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4')/children/child?$top=1") .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidXVpZDoxN2IwOWU5Ni00MTQxLTQzZjUtOWE3MC02MTFlYjBlOGY2YjQiLCJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); + body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); }) ])))))); @@ -1363,7 +1363,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D', value: [{ __id: '52eff9ea82550183880b9d64c20487642fa6e60c', '__Submissions-id': 'rtwo', @@ -1454,7 +1454,7 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body.value[0].name.should.be.eql('Candace'); - body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIiwicmVwZWF0SWQiOiIzMjgwOWFlMmIzZGM0MDRlYTI5MjIwNWViODg0YjIxZmE0ZTlhY2M1In0%3D'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjMyODA5YWUyYjNkYzQwNGVhMjkyMjA1ZWI4ODRiMjFmYTRlOWFjYzUifQ%3D%3D'); return body['@odata.nextLink']; }); @@ -1464,7 +1464,7 @@ describe('api: /forms/:id.svc', () => { body.value[0].name.should.be.eql('Billy'); - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D" // should.not.exist(body['@odata.nextLink']); }); })); @@ -1480,7 +1480,7 @@ describe('api: /forms/:id.svc', () => { .expect(200) .then(({ body }) => { body.value[0].name.should.be.eql('Billy'); - body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D'); return body['@odata.nextLink']; }); @@ -1490,7 +1490,7 @@ describe('api: /forms/:id.svc', () => { body.value[0].name.should.be.eql('Blaine'); - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byIsInJlcGVhdElkIjoiNTJlZmY5ZWE4MjU1MDE4Mzg4MGI5ZDY0YzIwNDg3NjQyZmE2ZTYwYyJ9" + // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D" should.not.exist(body['@odata.nextLink']); }); diff --git a/test/unit/formats/odata.js b/test/unit/formats/odata.js index 9e9409bb6..d60bb686a 100644 --- a/test/unit/formats/odata.js +++ b/test/unit/formats/odata.js @@ -746,7 +746,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6ImM3NmQwY2NjNmQ1ZGEyMzZiZTdiOTNiOTg1YTgwNDEzZDJlM2UxNzIifQ%3D%3D', value: [{ __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', '__Submissions-id': 'two', @@ -803,7 +803,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjNzZkMGNjYzZkNWRhMjM2YmU3YjkzYjk4NWE4MDQxM2QyZTNlMTcyIn0%3D', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImM3NmQwY2NjNmQ1ZGEyMzZiZTdiOTNiOTg1YTgwNDEzZDJlM2UxNzIifQ%3D%3D', value: [{ __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', '__Submissions-id': 'two', @@ -928,7 +928,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImNmOWExYjVjYzgzYzZkNjI3MGMxZWI5ODg2MGQyOTRlYWM1ZDUyNmQifQ%3D%3D"); }); }); }); @@ -941,7 +941,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1&$wkt=true", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=01eyJpbnN0YW5jZUlkIjoidHdvIiwicmVwZWF0SWQiOiJjZjlhMWI1Y2M4M2M2ZDYyNzBjMWViOTg4NjBkMjk0ZWFjNWQ1MjZkIn0%3D"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=01eyJyZXBlYXRJZCI6ImNmOWExYjVjYzgzYzZkNjI3MGMxZWI5ODg2MGQyOTRlYWM1ZDUyNmQifQ%3D%3D"); }); }); }); @@ -1038,7 +1038,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiI4ZDJkYzdiZDNlOTdhNjkwYzA4MTNlNjQ2NjU4ZTUxMDM4ZWI0MTQ0In0%3D", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6IjhkMmRjN2JkM2U5N2E2OTBjMDgxM2U2NDY2NThlNTEwMzhlYjQxNDQifQ%3D%3D", value: [{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -1091,7 +1091,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJpbnN0YW5jZUlkIjoiZG91YmxlIiwicmVwZWF0SWQiOiJiNzE2ZGQ4Yjc5YTRjOTM2OWQ2YjFlN2E5YzlkNTVhYzE4ZGExMzE5In0%3D", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6ImI3MTZkZDhiNzlhNGM5MzY5ZDZiMWU3YTljOWQ1NWFjMThkYTEzMTkifQ%3D%3D", value: [{ __id: '8d2dc7bd3e97a690c0813e646658e51038eb4144', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', From 72a3d84f27c6691b9df8e5d62095907959e0dac5 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Wed, 30 Aug 2023 20:48:19 -0400 Subject: [PATCH 07/10] fix entity count --- lib/model/query/entities.js | 2 ++ lib/resources/odata.js | 6 ++-- test/integration/api/odata-entities.js | 50 ++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/lib/model/query/entities.js b/lib/model/query/entities.js index 7c5d1ecb3..690a1db52 100644 --- a/lib/model/query/entities.js +++ b/lib/model/query/entities.js @@ -335,6 +335,7 @@ SELECT * FROM ( SELECT count(*) count FROM entities WHERE "datasetId" = ${datasetId} + AND "deletedAt" IS NULL AND ${odataFilter(options.filter, odataToColumnMap)} ) AS "all" @@ -347,6 +348,7 @@ CROSS JOIN ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id `: sql``} WHERE "datasetId" = ${datasetId} + AND "deletedAt" IS NULL AND ${odataFilter(options.filter, odataToColumnMap)} ) AS skiptoken`); diff --git a/lib/resources/odata.js b/lib/resources/odata.js index efdf3b1fc..830b8d36e 100644 --- a/lib/resources/odata.js +++ b/lib/resources/odata.js @@ -62,12 +62,14 @@ module.exports = (service, endpoint) => { service.get(`${base}/:table`, endpoint.odata.json(({ Forms, Submissions, env }, { auth, params, originalUrl, query }) => getForm(Forms, auth, params) .then((form) => { + const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) const options = QueryOptions.fromODataRequest(params, query); - return Promise.all([ + return Promise.all([ Forms.getFields(form.def.id).then(selectFields(query, params.table)), Submissions.streamForExport(form.id, draft, undefined, options), ((params.table === 'Submissions') && options.hasPaging()) - ? Submissions.countByFormId(form.id, draft, options) : resolve({}) + ? Submissions.countByFormId(form.id, draft, options) : resolve({}), + delay(2000) ]) .then(([fields, stream, { count, remaining }]) => json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count, remaining))); diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index 0ef6a1064..fdcf169a0 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -147,7 +147,7 @@ describe('api: /datasets/:name.svc', () => { await createSubmissions(asAlice, container, 2); - const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1') + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1&$count=true') .expect(200) .then(({ body }) => { body.value[0].age.should.be.eql('2'); @@ -155,7 +155,8 @@ describe('api: /datasets/:name.svc', () => { uuid: body.value[0].__id, }; const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); - body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24skiptoken=${token}`); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24count=true&%24skiptoken=${token}`); + body['@odata.count'].should.be.eql(2); return body['@odata.nextLink']; }); @@ -166,6 +167,51 @@ describe('api: /datasets/:name.svc', () => { .expect(200) .then(({ body }) => { body.value[0].age.should.be.eql('1'); + body['@odata.count'].should.be.eql(4); + should.not.exist(body['@odata.nextLink']); + }); + })); + + it('should not return deleted entities - opaque cursor', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 5); + + const uuids = await asAlice.get('/v1/projects/1/datasets/people/entities') + .then(({body}) => body.map(e => e.uuid)); + + + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=2&$count=true') + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('5'); + body.value[1].age.should.be.eql('4'); + const tokenData = { + uuid: body.value[1].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=2&%24count=true&%24skiptoken=${token}`); + body['@odata.count'].should.be.eql(5); + return body['@odata.nextLink']; + }); + + // let's delete submissions + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[0]}`) + .expect(200); + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[4]}`) + .expect(200); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('3'); + body.value[1].age.should.be.eql('2'); + body['@odata.count'].should.be.eql(3); should.not.exist(body['@odata.nextLink']); }); From c6b952433c127f09bbdcd0dfef006588115ed009 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Fri, 1 Sep 2023 17:18:19 -0400 Subject: [PATCH 08/10] datasets/:name is extendable --- lib/resources/datasets.js | 4 ++-- lib/resources/odata.js | 6 ++---- test/integration/api/odata-entities.js | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/resources/datasets.js b/lib/resources/datasets.js index c22ffd423..aa99ef8f0 100644 --- a/lib/resources/datasets.js +++ b/lib/resources/datasets.js @@ -22,8 +22,8 @@ module.exports = (service, endpoint) => { .then((project) => auth.canOrReject('dataset.list', project)) .then(() => Datasets.getList(params.id, queryOptions)))); - service.get('/projects/:projectId/datasets/:name', endpoint(({ Datasets }, { params, auth }) => - Datasets.get(params.projectId, params.name) + service.get('/projects/:projectId/datasets/:name', endpoint(({ Datasets }, { params, auth, queryOptions }) => + Datasets.get(params.projectId, params.name, true, queryOptions.extended) .then(getOrNotFound) .then((dataset) => auth.canOrReject('dataset.read', dataset) .then(() => Datasets.getMetadata(dataset))))); diff --git a/lib/resources/odata.js b/lib/resources/odata.js index 830b8d36e..efdf3b1fc 100644 --- a/lib/resources/odata.js +++ b/lib/resources/odata.js @@ -62,14 +62,12 @@ module.exports = (service, endpoint) => { service.get(`${base}/:table`, endpoint.odata.json(({ Forms, Submissions, env }, { auth, params, originalUrl, query }) => getForm(Forms, auth, params) .then((form) => { - const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) const options = QueryOptions.fromODataRequest(params, query); - return Promise.all([ + return Promise.all([ Forms.getFields(form.def.id).then(selectFields(query, params.table)), Submissions.streamForExport(form.id, draft, undefined, options), ((params.table === 'Submissions') && options.hasPaging()) - ? Submissions.countByFormId(form.id, draft, options) : resolve({}), - delay(2000) + ? Submissions.countByFormId(form.id, draft, options) : resolve({}) ]) .then(([fields, stream, { count, remaining }]) => json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count, remaining))); diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index fdcf169a0..13109424a 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -183,8 +183,7 @@ describe('api: /datasets/:name.svc', () => { await createSubmissions(asAlice, container, 5); const uuids = await asAlice.get('/v1/projects/1/datasets/people/entities') - .then(({body}) => body.map(e => e.uuid)); - + .then(({ body }) => body.map(e => e.uuid)); const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=2&$count=true') .expect(200) From 9e03d7a7bb5bf2c5ae4cfe88c3b3195b4bd6a57e Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Fri, 1 Sep 2023 17:28:08 -0400 Subject: [PATCH 09/10] added test for extended datasets/:name --- test/integration/api/datasets.js | 69 +++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/test/integration/api/datasets.js b/test/integration/api/datasets.js index baca3d2c9..f1e1a4d9d 100644 --- a/test/integration/api/datasets.js +++ b/test/integration/api/datasets.js @@ -556,18 +556,67 @@ describe('datasets and entities', () => { sourceForms.should.be.eql([ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, - { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } ]); + { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }]); properties.map(({ publishedAt, ...p }) => { publishedAt.should.be.isoDate(); return p; }).should.be.eql([ - { name: 'first_name', odataName: 'first_name', forms: [ - { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, - { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } - ] }, - { name: 'the.age', odataName: 'the_age', forms: [ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, ] }, - { name: 'address', odataName: 'address', forms: [ { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }, ] } + { + name: 'first_name', odataName: 'first_name', forms: [ + { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, + { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } + ] + }, + { name: 'the.age', odataName: 'the_age', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' },] }, + { name: 'address', odataName: 'address', forms: [{ name: 'simpleEntity2', xmlFormId: 'simpleEntity2' },] } + ]); + + }); + + })); + + it('should return the extended metadata of the dataset', testService(async (service) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.simpleEntity) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-111111111aaa', + label: 'Johnny Doe' + }) + .expect(200); + + await asAlice.get('/v1/projects/1/datasets/people') + .set('X-Extended-Metadata', 'true') + .expect(200) + .then(({ body }) => { + + const { createdAt, properties, lastEntity, ...ds } = body; + + ds.should.be.eql({ + name: 'people', + projectId: 1, + approvalRequired: false, + entities: 1, + linkedForms: [], + sourceForms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }] + }); + + lastEntity.should.be.recentIsoDate(); + + createdAt.should.be.recentIsoDate(); + + properties.map(({ publishedAt, ...p }) => { + publishedAt.should.be.isoDate(); + return p; + }).should.be.eql([ + { name: 'first_name', odataName: 'first_name', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }] }, + { name: 'age', odataName: 'age', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' },] } ]); }); @@ -904,7 +953,7 @@ describe('datasets and entities', () => { await asAlice.get('/v1/projects/1/datasets/people') .expect(200) .then(({ body }) => { - body.sourceForms.should.be.eql([ { name: 'simpleEntity', xmlFormId: 'simpleEntity' } ]); + body.sourceForms.should.be.eql([{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }]); }); })); @@ -2364,7 +2413,7 @@ describe('datasets and entities', () => { await Audits.getLatestByAction('dataset.update') .then(o => o.get()) - .then(audit => audit.details.should.eql({ properties: ['first_name', 'age', 'color_name', ] })); + .then(audit => audit.details.should.eql({ properties: ['first_name', 'age', 'color_name',] })); })); @@ -2399,7 +2448,7 @@ describe('datasets and entities', () => { container.oneFirst(sql`select count(*) from form_fields as fs join forms as f on fs."formId" = f.id where f."xmlFormId"='simpleEntity'`), container.oneFirst(sql`select count(*) from ds_property_fields`), ]) - .then((counts) => counts.should.eql([ 2, 6, 6 ])); + .then((counts) => counts.should.eql([2, 6, 6])); })); }); From a806eb97127b3c2c361966ab3a3a5cd7dc751ab0 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Tue, 12 Sep 2023 12:28:35 -0400 Subject: [PATCH 10/10] polish some of the tests --- test/integration/api/odata-entities.js | 17 ++++--- test/integration/api/odata.js | 62 +++++++++++++++++++------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index 13109424a..76211d2b2 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -199,18 +199,19 @@ describe('api: /datasets/:name.svc', () => { return body['@odata.nextLink']; }); - // let's delete submissions + // let's delete entities await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[0]}`) .expect(200); + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[2]}`) + .expect(200); await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[4]}`) .expect(200); await asAlice.get(nextlink.replace('http://localhost:8989', '')) .expect(200) .then(({ body }) => { - body.value[0].age.should.be.eql('3'); - body.value[1].age.should.be.eql('2'); - body['@odata.count'].should.be.eql(3); + body.value[0].age.should.be.eql('2'); + body['@odata.count'].should.be.eql(2); should.not.exist(body['@odata.nextLink']); }); @@ -240,6 +241,7 @@ describe('api: /datasets/:name.svc', () => { it('should return filtered entities with pagination', testService(async (service, container) => { const asAlice = await service.login('alice'); + const asBob = await service.login('bob'); await asAlice.post('/v1/projects/1/forms?publish=true') .set('Content-Type', 'application/xml') @@ -248,11 +250,11 @@ describe('api: /datasets/:name.svc', () => { await createSubmissions(asAlice, container, 2); - await container.run(sql`UPDATE entities SET "createdAt" = '2020-01-01'`); + await createSubmissions(asBob, container, 2, 2); - await createSubmissions(asAlice, container, 2, 2); + const bobId = await asBob.get('/v1/users/current').then(({ body }) => body.id); - const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1&$filter=__system/createdAt gt 2021-01-01') + const nextlink = await asAlice.get(`/v1/projects/1/datasets/people.svc/Entities?$top=1&$filter=__system/creatorId eq ${bobId}`) .expect(200) .then(({ body }) => { body.value.length.should.be.eql(1); @@ -264,6 +266,7 @@ describe('api: /datasets/:name.svc', () => { .expect(200) .then(({ body }) => { body.value[0].age.should.be.eql('3'); + should.not.exist(body['@odata.nextLink']); }); })); diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 7eb88033d..22adf5c67 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -322,7 +322,7 @@ describe('api: /forms/:id.svc', () => { it('should return just a count if asked', testService((service) => withSubmission(service, (asAlice) => asAlice.get("/v1/projects/1/forms/doubleRepeat.svc/Submissions('double')/children/child?$top=0&$count=true") - // .expect(200) + .expect(200) .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', @@ -725,7 +725,7 @@ describe('api: /forms/:id.svc', () => { const asAlice = await withSubmissions(service, identity); await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1') - // .expect(200) + .expect(200) .then(({ body }) => { const tokenData = { instanceId: body.value[0].__id, @@ -764,6 +764,43 @@ describe('api: /forms/:id.svc', () => { }); })); + it('should have no impact on skipToken when a new submission is created', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=2') + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rthree'); + body.value[1].__id.should.be.eql('rtwo'); + + const tokenData = { + instanceId: body.value[1].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + + const expectedNextLink = `http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=2&%24skiptoken=${(token)}`; + body['@odata.nextLink'].should.eql(expectedNextLink); + return body['@odata.nextLink']; + }); + + await asAlice.post('/v1/projects/1/forms/withrepeat/submissions') + .send(testData.instances.withrepeat.one + .replace('one', 'four') + .replace('Alice', 'John')) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rone'); + + should.not.exist(body['@odata.nextLink']); + }); + })); + it('should limit and filter Submissions', testService(async (service) => { const asAlice = await withSubmissions(service, identity); @@ -772,7 +809,7 @@ describe('api: /forms/:id.svc', () => { .expect(200); await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$filter=not __system/reviewState eq \'rejected\'') - // .expect(200) + .expect(200) .then(({ body }) => { const tokenData = { instanceId: body.value[0].__id, @@ -782,11 +819,11 @@ describe('api: /forms/:id.svc', () => { }); })); - it('should limit Submissions', testService(async (service) => { + it('should limit and return selected fields of Submissions', testService(async (service) => { const asAlice = await withSubmissions(service, identity); await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$select=age') - // .expect(200) + .expect(200) .then(({ body }) => { body.value[0].should.be.eql({ age: 38, @@ -1450,22 +1487,20 @@ describe('api: /forms/:id.svc', () => { it('should limit subtable results', testService(async (service) => { const asAlice = await withSubmissions(service, identity); - const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=1') + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=2') .expect(200) .then(({ body }) => { body.value[0].name.should.be.eql('Candace'); - body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjMyODA5YWUyYjNkYzQwNGVhMjkyMjA1ZWI4ODRiMjFmYTRlOWFjYzUifQ%3D%3D'); + body.value[1].name.should.be.eql('Billy'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D'); return body['@odata.nextLink']; }); await asAlice.get(nextlink.replace('http://localhost:8989', '')) .expect(200) .then(({ body }) => { - - body.value[0].name.should.be.eql('Billy'); - - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D" - // should.not.exist(body['@odata.nextLink']); + body.value[0].name.should.be.eql('Blaine'); + should.not.exist(body['@odata.nextLink']); }); })); @@ -1487,10 +1522,7 @@ describe('api: /forms/:id.svc', () => { await asAlice.get(nextlink.replace('http://localhost:8989', '')) .expect(200) .then(({ body }) => { - body.value[0].name.should.be.eql('Blaine'); - - // "http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D" should.not.exist(body['@odata.nextLink']); });