diff --git a/bin/validatorWorker.js b/bin/validatorWorker.js index 4956f530..c905a613 100755 --- a/bin/validatorWorker.js +++ b/bin/validatorWorker.js @@ -43,17 +43,30 @@ adapter process.exit(1) }) -function allChannelsTick() { - return ( - fetch(`${argv.sentryUrl}/channel/list?validator=${adapter.whoami()}`, { - timeout: cfg.LIST_TIMEOUT - }) - .then(res => res.json()) - .then(({ channels }) => Promise.all(channels.map(validatorTick))) - .then(allResults => logPostChannelsTick(allResults)) - // eslint-disable-next-line no-console - .catch(e => console.error('allChannelsTick failed', e)) +function getChannels(pageNumber) { + const page = pageNumber && `&page=${pageNumber}` + const defaultUrl = `${argv.sentryUrl}/channel/list?validator=${adapter.whoami()}` + const url = (page && `${defaultUrl}${page}`) || defaultUrl + + return fetch(url, { + timeout: cfg.LIST_TIMEOUT + }).then(res => res.json()) +} + +async function allChannelsTick(currentPage) { + const { channels, total, page } = await getChannels(currentPage) + + const allResults = await Promise.all(channels.map(validatorTick)).catch(e => + logger.error('allChannelsTick failed', e) ) + logPostChannelsTick(allResults) + + if (total <= page) return null + + const nextPage = parseInt(page, 10) + 1 + return Promise.all([allChannelsTick(nextPage), wait(cfg.WAIT_TIME)]) + .then(function() {}) + .catch(e => logger.error(`tick for next page ${nextPage} failed`, e)) } function loopChannels() { diff --git a/routes/channel.js b/routes/channel.js index 1520ffca..ab6e6a41 100644 --- a/routes/channel.js +++ b/routes/channel.js @@ -81,10 +81,13 @@ function getEventAggregates(req, res, next) { .catch(next) } -function getList(req, res, next) { +async function getList(req, res, next) { const { CHANNELS_FIND_LIMIT } = cfg + // assign 0 default value + const { page = 0 } = req.query + const channelsCol = db.getMongo().collection('channels') - const skip = req.query.page && parseInt(req.query.page, 10) * CHANNELS_FIND_LIMIT + const skip = page && parseInt(page, 10) * CHANNELS_FIND_LIMIT let query = { validUntil: { $gt: Math.floor(Date.now() / 1000) } } @@ -93,13 +96,18 @@ function getList(req, res, next) { // this query will find anything where the array contains an object with this ID query = { ...query, 'spec.validators.id': req.query.validator } } + + const channelTotal = await channelsCol.countDocuments(query) + // subtract one becuase page counting starts from 0 + const totalPages = Math.ceil(channelTotal / CHANNELS_FIND_LIMIT) - 1 + return channelsCol .find(query, { projection: { _id: 0 } }) .limit(CHANNELS_FIND_LIMIT) .skip(skip || 0) .toArray() .then(function(channels) { - res.send({ channels }) + res.send({ channels, total: totalPages, page }) }) .catch(next) } diff --git a/routes/schemas.js b/routes/schemas.js index 726cf331..143ddf2f 100644 --- a/routes/schemas.js +++ b/routes/schemas.js @@ -51,8 +51,7 @@ module.exports = { adUnits: Joi.array().items(Joi.object()), targeting: Joi.array().items(Joi.object()), validators: validators(cfg), - withdrawPeriodStart: Joi.number() - .required(), + withdrawPeriodStart: Joi.number().required(), minPerImpression: numericString.default('1'), maxPerImpression: numericString.default('1'), nonce: Joi.string(),