feat: már lementett számlák nyilvántartása, átugrása

juzraai · Oct 2, 2019 · 337cb4e · 337cb4e
1 parent c791bbd
commit 337cb4e
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/src/lib.js b/src/lib.js
@@ -108,13 +108,15 @@ function parse_szamla_list(body) {
 	const providerIndex = indexOfOrThrowError('Szolgaltato');
 	const customNameIndex = indexOfOrThrowError('Szamlakibocsatoi azonosito');
 	const dateIndex = indexOfOrThrowError('Kiallitas datuma');
+	const billIdIndex = indexOfOrThrowError('Szamlaszam Bizonylatszam');
 	const invoices = [];
 	$('.szamla_table tbody tr').each((i, tr) => {
 		const invoice = {
 			rowid: $(tr).html().toString().match(/rowid=(\d+)/)[1],
 			provider: normalize($(tr.childNodes[providerIndex]).text()),
 			customName: normalize($(tr.childNodes[customNameIndex]).text()),
-			date: $(tr.childNodes[dateIndex]).text()
+			date: $(tr.childNodes[dateIndex]).text(),
+			billId: $(tr.childNodes[billIdIndex]).text()
 		};
 		invoices.push(invoice);
 	});

diff --git a/src/main.js b/src/main.js
@@ -1,4 +1,5 @@
 require('dotenv').config();
+const fs = require('fs');
 const path = require('path');
 const mkdirp = require('util').promisify(require('mkdirp'));
 const dijnet = require('./lib');
@@ -11,6 +12,24 @@ function tmp(name) {
 	return process.env.TEMP_DIR.length === 0 ? null : path.join(process.env.TEMP_DIR, name);
 }
 
+const alreadyCrawledIdsFile = path.join(process.env.OUTPUT_DIR, 'kesz.txt');
+let alreadyCrawledIds = null;
+function isAlreadyCrawled(id) {
+	if (null == alreadyCrawledIds) {
+
+		if (fs.existsSync(alreadyCrawledIdsFile)) {
+			alreadyCrawledIds = fs.readFileSync(alreadyCrawledIdsFile, 'utf8').split('\n');
+		} else {
+			alreadyCrawledIds = [];
+		}
+	}
+	return alreadyCrawledIds.includes(id);
+}
+
+function markAlreadyCrawled(id) {
+	fs.appendFileSync(alreadyCrawledIdsFile, id + '\n');
+}
+
 const start = async () => {
 	try {
 		log.success('Díjnet-bot indul');
@@ -39,6 +58,12 @@ const start = async () => {
 
 		for (let i = 0; i < invoices.length; i++) {
 			const invoice = invoices[i];
+
+			if (isAlreadyCrawled(invoice.billId)) {
+				log.success('[%d/%d] Számla #%d már letöltve, most kihagyjuk', i + 1, invoices.length, invoice.rowid);
+				continue;
+			}
+
 			const dir = path.join(process.env.OUTPUT_DIR, `${invoice.provider} - ${invoice.customName}`, invoice.date);
 
 			log.info('[%d/%d] Számla #%d kiválasztása', i + 1, invoices.length, invoice.rowid);
@@ -57,6 +82,7 @@ const start = async () => {
 				await dijnet.download(file, dir);
 			}
 
+			markAlreadyCrawled(invoice.billId);
 			log.success('[%d/%d] Számla #%d fájljai (%d db) lementve', i + 1, invoices.length, invoice.rowid, files.length);
 			log.info('Visszatérés a számla listához');
 			await dijnet.sleep(3);