feature: validatorWorker ability to process more channels than CHANNE…

…L_FIND_LIMIT

bin/validatorWorker.js

@@ -43,17 +43,30 @@ adapter
 		process.exit(1)
 	})
 
-function allChannelsTick() {
-	return (
-		fetch(`${argv.sentryUrl}/channel/list?validator=${adapter.whoami()}`, {
-			timeout: cfg.LIST_TIMEOUT
-		})
-			.then(res => res.json())
-			.then(({ channels }) => Promise.all(channels.map(validatorTick)))
-			.then(allResults => logPostChannelsTick(allResults))
-			// eslint-disable-next-line no-console
-			.catch(e => console.error('allChannelsTick failed', e))
+function getChannels(pageNumber) {
+	const page = pageNumber && `&page=${pageNumber}`
+	const defaultUrl = `${argv.sentryUrl}/channel/list?validator=${adapter.whoami()}`
+	const url = (page && `${defaultUrl}${page}`) || defaultUrl
+
+	return fetch(url, {
+		timeout: cfg.LIST_TIMEOUT
+	}).then(res => res.json())
+}
+
+async function allChannelsTick(currentPage) {
+	const { channels, total, page } = await getChannels(currentPage)
+
+	const allResults = await Promise.all(channels.map(validatorTick)).catch(e =>
+		logger.error('allChannelsTick failed', e)
 	)
+	logPostChannelsTick(allResults)
+
+	if (total <= page) return null
+
+	const nextPage = parseInt(page, 10) + 1
+	return Promise.all([allChannelsTick(nextPage), wait(cfg.WAIT_TIME)])
+		.then(function() {})
+		.catch(e => logger.error(`tick for next page ${nextPage} failed`, e))
 }
 
 function loopChannels() {

routes/channel.js

@@ -81,10 +81,13 @@ function getEventAggregates(req, res, next) {
 		.catch(next)
 }
 
-function getList(req, res, next) {
+async function getList(req, res, next) {
 	const { CHANNELS_FIND_LIMIT } = cfg
+	// assign 0 default value
+	const { page = 0 } = req.query
+
 	const channelsCol = db.getMongo().collection('channels')
-	const skip = req.query.page && parseInt(req.query.page, 10) * CHANNELS_FIND_LIMIT
+	const skip = page && parseInt(page, 10) * CHANNELS_FIND_LIMIT
 	let query = {
 		validUntil: { $gt: Math.floor(Date.now() / 1000) }
 	}
@@ -93,13 +96,18 @@ function getList(req, res, next) {
 		// this query will find anything where the array contains an object with this ID
 		query = { ...query, 'spec.validators.id': req.query.validator }
 	}
+
+	const channelTotal = await channelsCol.countDocuments(query)
+	// subtract one becuase page counting starts from 0
+	const totalPages = Math.ceil(channelTotal / CHANNELS_FIND_LIMIT) - 1
+
 	return channelsCol
 		.find(query, { projection: { _id: 0 } })
 		.limit(CHANNELS_FIND_LIMIT)
 		.skip(skip || 0)
 		.toArray()
 		.then(function(channels) {
-			res.send({ channels })
+			res.send({ channels, total: totalPages, page })
 		})
 		.catch(next)
 }

routes/schemas.js

@@ -51,8 +51,7 @@ module.exports = {
 			adUnits: Joi.array().items(Joi.object()),
 			targeting: Joi.array().items(Joi.object()),
 			validators: validators(cfg),
-			withdrawPeriodStart: Joi.number()
-				.required(),
+			withdrawPeriodStart: Joi.number().required(),
 			minPerImpression: numericString.default('1'),
 			maxPerImpression: numericString.default('1'),
 			nonce: Joi.string(),