From 6c95ad1639596717c726f51b6605322f9a5efe1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zsolt=20Jur=C3=A1nyi?= Date: Tue, 22 Oct 2019 22:40:30 +0200 Subject: [PATCH] =?UTF-8?q?refactor:=20=C3=BAjra=C3=ADrtam=20az=20eg=C3=A9?= =?UTF-8?q?sz=20cuccot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: lib.js szétdarabolva osztályokra, process.env helyett Config modellt használunk már, új logger.js - részletek: ld. JSDoc --- .env.example | 12 ++- .gitignore | 1 + index.js | 9 ++- package-lock.json | 73 +++++++++--------- package.json | 3 +- src/bill-file.js | 15 ++++ src/bill.js | 31 ++++++++ src/browser.js | 50 ++++++++++++ src/cli.js | 44 ++++------- src/conf.js | 21 ------ src/config.js | 33 ++++++++ src/configurator.js | 87 +++++++++++++++++++++ src/dijnet-agent.js | 104 +++++++++++++++++++++++++ src/dijnet-browser.js | 126 +++++++++++++++++++++++++++++++ src/dijnet-parser.js | 75 ++++++++++++++++++ src/err.js | 20 ----- src/lib.js | 156 -------------------------------------- src/logger.js | 172 ++++++++++++++++++++++++++++-------------- src/main.js | 146 ++++++++++++----------------------- src/repo.js | 80 ++++++++++++++++++++ 20 files changed, 830 insertions(+), 428 deletions(-) create mode 100644 src/bill-file.js create mode 100644 src/bill.js create mode 100644 src/browser.js delete mode 100644 src/conf.js create mode 100644 src/config.js create mode 100644 src/configurator.js create mode 100644 src/dijnet-agent.js create mode 100644 src/dijnet-browser.js create mode 100644 src/dijnet-parser.js delete mode 100644 src/err.js delete mode 100644 src/lib.js create mode 100644 src/repo.js diff --git a/.env.example b/.env.example index b5899f1..47ed1e0 100644 --- a/.env.example +++ b/.env.example @@ -20,10 +20,8 @@ OUTPUT_DIR=./szamlak SLEEP=3 -# Naplózás szintje: -# 0 = semmit nem ír a képernyőre -# 1 = hibaüzenetek -# 2 = 1 + sikeres műveletsorok jelzése -# 3 = 2 + műveletek -# 4 = 3 + részletesebb információk -LOG_LEVEL=2 \ No newline at end of file +# Naplózás módja: +# default = terminálban könnyen érthető folyamatjelző, fájlba irányítva bővített napló (korábban: LOG_LEVEL=1/2) +# verbose = bővített napló, minden műveletről tájékoztat (korábban: LOG_LEVEL=4) +# quiet = nincs kimenet (korábban: LOG_LEVEL=0) +LOG_MODE=default \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2bc350f..62dcff0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ szamlak/ temp/ .env error.log +src/_OLD \ No newline at end of file diff --git a/index.js b/index.js index be80bd5..eec0217 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,13 @@ #!/usr/bin/env node const SingleInstance = require('single-instance'); +const Logger = require('./src/logger'); const { start } = require('./src/main'); -const { handleError } = require('./src/err'); new SingleInstance('dijnet-bot').lock().then(start).catch(error => { - handleError(error.stack ? error : 'A Díjnet bot már fut, és nem futtatható több példányban.'); + try { + new Logger().error(error.stack ? error : 'A Díjnet bot már fut, és nem futtatható több példányban.'); + } catch (error2) { // in case Logger is also dead + console.log(error2); + } + process.exit(1); }); diff --git a/package-lock.json b/package-lock.json index 5b5aaff..ab17e7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, "requires": { "color-convert": "^1.9.0" } @@ -126,6 +127,7 @@ "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, "requires": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", @@ -168,6 +170,7 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, "requires": { "color-name": "1.1.3" } @@ -175,7 +178,8 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true }, "commander": { "version": "3.0.2", @@ -640,6 +644,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, "requires": { "is-arrayish": "^0.2.1" } @@ -647,20 +652,14 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", - "requires": { - "escape-string-regexp": "^1.0.5" - } + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true }, "find-up": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, "requires": { "locate-path": "^2.0.0" } @@ -1030,7 +1029,8 @@ "graceful-fs": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.2.tgz", - "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==" + "integrity": "sha512-IItsdsea19BoLC7ELy13q1iJFNmd7ofZH5+X/pJr90/nRoPEX0DJo1dHDbgtYWOhJhcCgMDTOw84RZ72q6lB+Q==", + "dev": true }, "handlebars": { "version": "4.4.0", @@ -1047,7 +1047,8 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true }, "hosted-git-info": { "version": "2.8.4", @@ -1098,7 +1099,8 @@ "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true }, "is-finite": { "version": "1.0.2", @@ -1156,7 +1158,8 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true }, "json-stringify-safe": { "version": "5.0.1", @@ -1178,10 +1181,16 @@ "json-buffer": "3.0.0" } }, + "kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==" + }, "load-json-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, "requires": { "graceful-fs": "^4.1.2", "parse-json": "^4.0.0", @@ -1193,6 +1202,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, "requires": { "p-locate": "^2.0.0", "path-exists": "^3.0.0" @@ -1412,6 +1422,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, "requires": { "p-try": "^1.0.0" } @@ -1420,6 +1431,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, "requires": { "p-limit": "^1.1.0" } @@ -1427,7 +1439,8 @@ "p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=" + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true }, "parse-github-repo-url": { "version": "1.4.1", @@ -1439,6 +1452,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, "requires": { "error-ex": "^1.3.1", "json-parse-better-errors": "^1.0.1" @@ -1455,7 +1469,8 @@ "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true }, "path-parse": { "version": "1.0.6", @@ -1475,7 +1490,8 @@ "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true }, "pinkie": { "version": "2.0.4", @@ -1492,15 +1508,6 @@ "pinkie": "^2.0.0" } }, - "pkg-conf": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-2.1.0.tgz", - "integrity": "sha1-ISZRTKbyq/69FoWW3xi6V4Z/AFg=", - "requires": { - "find-up": "^2.0.0", - "load-json-file": "^4.0.0" - } - }, "prepend-http": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", @@ -1650,16 +1657,6 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true }, - "signale": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/signale/-/signale-1.4.0.tgz", - "integrity": "sha512-iuh+gPf28RkltuJC7W5MRi6XAjTDCAPC/prJUpQoG4vIP3MJZ+GTydVnodXA7pwvTKb2cA0m9OFZW/cdWy/I/w==", - "requires": { - "chalk": "^2.3.2", - "figures": "^2.0.0", - "pkg-conf": "^2.1.0" - } - }, "single-instance": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/single-instance/-/single-instance-0.0.1.tgz", @@ -1885,7 +1882,8 @@ "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=" + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true }, "strip-indent": { "version": "2.0.0", @@ -1897,6 +1895,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "requires": { "has-flag": "^3.0.0" } diff --git a/package.json b/package.json index d15ebc1..1d0ce0b 100644 --- a/package.json +++ b/package.json @@ -36,14 +36,13 @@ }, "homepage": "https://github.com/juzraai/dijnet-bot#readme", "dependencies": { - "chalk": "^2.4.2", "cheerio": "^1.0.0-rc.3", "commander": "^3.0.2", "dotenv": "^8.0.0", "got": "^9.6.0", + "kleur": "^3.0.3", "lodash.deburr": "^4.1.0", "mkdirp": "^0.5.1", - "signale": "^1.4.0", "single-instance": "0.0.1", "tough-cookie": "^3.0.0" }, diff --git a/src/bill-file.js b/src/bill-file.js new file mode 100644 index 0000000..4e278d8 --- /dev/null +++ b/src/bill-file.js @@ -0,0 +1,15 @@ +/** + * Represents a bill's downloadable file. + */ +class BillFile { + /** + * @param {string} name Display name + * @param {string} dijnetPath Díjnet path, relative to `DijnetBrowser`'s `baseUrl` + */ + constructor(name, dijnetPath) { + this.name = name; + this.dijnetPath = dijnetPath; + } +} + +module.exports = BillFile; diff --git a/src/bill.js b/src/bill.js new file mode 100644 index 0000000..feaf17d --- /dev/null +++ b/src/bill.js @@ -0,0 +1,31 @@ +/** + * Represents bill metadata. + */ +class Bill { + /** + * @param {Bill} data Bill metadata available on Díjnet + * @param {string} data.rowId Row ID in search results + * @param {string} data.serviceProvider Service provider + * @param {string} data.billIssuerId Bill issuer ID + * @param {string} data.billId Bill ID + * @param {string} data.dateOfIssue Date of issue + * @param {string} data.finalAmount Final amount + * @param {string} data.dueDate Due date + * @param {string} data.payable Payable + * @param {string} data.status Status + */ + constructor(data) { + data = data || {}; + this.rowId = data.rowId; + this.serviceProvider = data.serviceProvider; + this.billIssuerId = data.billIssuerId; + this.billId = data.billId; + this.dateOfIssue = data.dateOfIssue; + this.finalAmount = data.finalAmount; + this.dueDate = data.dueDate; + this.payable = data.payable; + this.status = data.status; + } +} + +module.exports = Bill; diff --git a/src/browser.js b/src/browser.js new file mode 100644 index 0000000..dcf6ded --- /dev/null +++ b/src/browser.js @@ -0,0 +1,50 @@ +const got = require('got'); +const { CookieJar } = require('tough-cookie'); + +/** + * Wrapper around `got` HTTP client. Adds a cookie jar and stores the last successful response. + */ +class Browser { + constructor() { + this.cookieJar = new CookieJar(); + /** @type {got.Response} */ + this.lastNavigationResponse = null; + } + + /** + * Sends a HTTP request to the given URL. + * + * @param {string} url Requested URL + * @param {got.GotJSONOptions} options Request options (method, headers, body, encoding, etc.) + * @returns {got.GotPromise} Response + */ + async request(url, options) { + return got(url, Object.assign({ cookieJar: this.cookieJar }, options)); + } + + /** + * Sends a GET request to the given URL, then stores the response in `lastNavigationResponse`. + * + * @param {string} url Requested URL + * @param {got.GotJSONOptions} options Request options (headers, encoding, etc.) + * @returns {got.GotPromise} Response + */ + async navigate(url, options) { + this.lastNavigationResponse = await this.request(url, Object.assign({ method: 'GET' }, options)); + return this.lastNavigationResponse; + } + + /** + * Sends a POST request to the given URL, then stores the response in `lastNavigationResponse`. + * + * @param {string} url Requested URL + * @param {got.GotJSONOptions} options Request options (body, headers, encoding, etc.) + * @returns {got.GotPromise} Response + */ + async submit(url, options) { + this.lastNavigationResponse = await this.request(url, Object.assign({ method: 'POST' }, options)); + return this.lastNavigationResponse; + } +} + +module.exports = Browser; diff --git a/src/cli.js b/src/cli.js index 7aee29d..e0cce55 100644 --- a/src/cli.js +++ b/src/cli.js @@ -1,14 +1,22 @@ -const program = require('commander'); +const commander = require('commander'); +const Config = require('./config'); // eslint-disable-line no-unused-vars -function config() { +/** + * Creates a Commander.js program definiton. + * + * @param {Config} defaultConfig Configuration to be used to print default values + * @returns {commander.Command} Commander.js program definiton + */ +function getCli(defaultConfig) { + const program = new commander.Command(); program .name('dijnet-bot') - .description('Automatikusan lementi az összes számládat. :)') + .description('Az összes számlád még egy helyen. :)') .option('-u, --user ', 'Díjnet felhasználónév') .option('-p, --pass ', 'Díjnet jelszó') - .option('-s, --sleep ', `A Díjnet kérések közötti szünet másodpercben (alapértelmezetten ${process.env.SLEEP})`, parseInt) - .option('-o, --output-dir ', `A kimeneti mappa útvonala, ahová a számlák kerülmek (alapértelmezetten ${process.env.OUTPUT_DIR})`) - .option('-t, --temp-dir ', 'Ha meg van adva, akkor ide menti ki a letöltött HTML lapokat, további kézi elemzés céljára') + .option('-s, --sleep ', `A Díjnet kérések közötti szünet másodpercben (alapértelmezetten ${defaultConfig.sleep})`, parseInt) + .option('-o, --output-dir ', `A kimeneti mappa útvonala, ahová a számlák kerülmek (alapértelmezetten ${defaultConfig.outputDir})`) + .option('-t, --temp-dir ', 'Ha meg van adva, akkor ide menti ki a letöltött HTML lapokat és cookie-kat, további kézi elemzés céljára') .option('-q, --quiet', 'Csendes mód, nem fog írni a képernyőre') .option('-v, --verbose', 'Részletesebb tájékoztatás a folyamatról (alacsonyabb prioritású, mint `-q`)') .helpOption('-h, --help', 'Megjeleníti ezeket a sorokat') @@ -23,27 +31,7 @@ function config() { console.log('DIJNET_PASS=jelszó'); console.log(''); }); - - program.parse(process.argv); - - process.env.DIJNET_USER = program.user || process.env.DIJNET_USER; - process.env.DIJNET_PASS = program.pass || process.env.DIJNET_PASS; - process.env.OUTPUT_DIR = program.outputDir || process.env.OUTPUT_DIR; - process.env.SLEEP = Math.max(program.sleep || process.env.SLEEP, 1); - process.env.TEMP_DIR = program.tempDir || process.env.TEMP_DIR; - if (program.verbose) { - process.env.LOG_LEVEL = 4; - } - if (program.quiet) { - process.env.LOG_LEVEL = 0; - } -} - -function printHelpAndExit() { - program.help(); + return program; } -module.exports = { - config, - printHelpAndExit -}; +module.exports = { getCli }; diff --git a/src/conf.js b/src/conf.js deleted file mode 100644 index 6e095cd..0000000 --- a/src/conf.js +++ /dev/null @@ -1,21 +0,0 @@ -const dotenv = require('dotenv'); -const cli = require('./cli'); - -function configurate() { - // Load environment variables and defaults - dotenv.config(); - process.env.LOG_LEVEL = process.env.LOG_LEVEL || 2; - process.env.OUTPUT_DIR = process.env.OUTPUT_DIR || './szamlak'; - process.env.SLEEP = process.env.SLEEP || 3; - process.env.TEMP_DIR = process.env.TEMP_DIR || ''; - - // Load command line arguments - cli.config(); - - // Auto print help - if (!process.env.DIJNET_USER || !process.env.DIJNET_PASS) { - cli.printHelpAndExit(); - } -} - -module.exports = configurate; diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..8df9698 --- /dev/null +++ b/src/config.js @@ -0,0 +1,33 @@ +/** + * Represents Dijnet Bot configuration. + */ +class Config { + /** + * @param {Config} data Configuration + * @param {string} doneFile Name of the file (relative to outputDir) where completed bills will be listed + * @param {string} errorFile Name of the file where error log will be written + * @param {string} outputDir Path to output directory where bills' files will be stored + * @param {string} pass Díjnet password + * @param {boolean} quiet Quiet mode, true means nothing will be written to standard output + * @param {number} sleep Amount of sleep before every request, in seconds + * @param {string} tempDir Path to temporary directory where HTML files should be written (optional) + * @param {boolean} tty Whether the script runs in an interactive terminal, false means standard output is being redirected + * @param {string} user Díjnet username + * @param {boolean} verbose Verbose mode, true means every operation will be logged to standard output + */ + constructor(data) { + data = data || {}; + this.doneFile = data.doneFile || 'kesz.txt'; // relative to outputDir! + this.errorFile = data.errorFile || './error.log'; + this.outputDir = data.outputDir || './szamlak'; + this.pass = data.pass; + this.quiet = data.quiet; + this.sleep = data.sleep || 3; + this.tempDir = data.tempDir; + this.tty = data.tty || process.stdout.isTTY; + this.user = data.user; + this.verbose = data.verbose; + } +} + +module.exports = Config; diff --git a/src/configurator.js b/src/configurator.js new file mode 100644 index 0000000..a638ce1 --- /dev/null +++ b/src/configurator.js @@ -0,0 +1,87 @@ +const dotenv = require('dotenv'); +const cli = require('./cli'); +const Config = require('./config'); + +/** + * Fetches configuration from environment variables (.env) and command line + * arguments in this order. If user or pass is missing at the end, + * it will print out help and exit. + * + * @returns {Config} Configuration + */ +async function getConfig() { + const config = new Config(); // default configuration + const program = cli.getCli(config); + + loadEnv(config); + loadArgs(program, config); + if (!config.user || !config.pass) { + program.help(); + } + + if (!config.tty && !config.quiet) { + config.verbose = true; + } + + return config; +} + +/** + * @param {Config} config Configuration to be updated + * @returns {Config} The same configuration which is also updated + */ +function loadEnv(config) { + dotenv.config(); + config.outputDir = process.env.OUTPUT_DIR || config.outputDir; + config.pass = process.env.DIJNET_PASS || config.pass; + + config.sleep = Math.max(parseInt(process.env.SLEEP || 0, 10), config.sleep); + config.tempDir = process.env.TEMP_DIR || config.tempDir; + config.user = process.env.DIJNET_USER || config.user; + + const logLevel = parseInt(process.env.LOG_LEVEL || 2, 10); // for backward compatibility + + const verboseV1 = logLevel > 2; + const verboseV2 = (process.env.LOG_MODE || '').toLowerCase() === 'verbose'; + if (verboseV1 || verboseV2) { + config.quiet = false; + config.verbose = true; + } + + const quietV1 = logLevel === 0; + const quietV2 = (process.env.LOG_MODE || '').toLowerCase() === 'quiet'; + if (quietV1 || quietV2) { + config.quiet = true; + config.verbose = false; + } + + return config; +} + +/** + * @param {commander.Command} program Commander.js program definiton + * @param {Config} config Configuration to be updated + * @returns {Config} The same configuration which is also updated + */ +function loadArgs(program, config) { + program.parse(process.argv); + + config.outputDir = program.outputDir || config.outputDir; + config.pass = program.pass || config.pass; + config.sleep = Math.max(parseInt(program.sleep || 0, 10), new Config().sleep); // env maybe modified, but CLI supposed to overwrite it, we have to maximize for default value + config.tempDir = program.tempDir || config.tempDir; + config.user = program.user || config.user; + + if (program.verbose) { + config.quiet = false; + config.verbose = true; + } + + if (program.quiet) { + config.quiet = true; + config.verbose = false; + } + return config; +} + +module.exports = { getConfig }; diff --git a/src/dijnet-agent.js b/src/dijnet-agent.js new file mode 100644 index 0000000..8e7689a --- /dev/null +++ b/src/dijnet-agent.js @@ -0,0 +1,104 @@ +const Config = require('./config'); +const DijnetBrowser = require('./dijnet-browser'); +const Logger = require('./logger'); + +/** + * Provides an API to Díjnet's functions / links. Knows which page depends on + * which another page, throws errors if functions called in wrong order. + */ +class DijnetAgent { + /** + * @param {Config} config Configuration + * @param {Logger} logger Logger + * @param {DijnetBrowser} dijnetBrowser DijnetBrowser + */ + constructor(config, logger, dijnetBrowser) { + this.config = config || new Config(); + this.logger = logger || new Logger(this.config); + this.browser = dijnetBrowser || new DijnetBrowser(this.config, this.logger); + } + + /** + * Logs in to Díjnet. + */ + async login() { + const body = `vfw_form=login_check_password&username=${this.config.user}&password=${this.config.pass}`; + await this.browser.submit('/login/login_check_password', body); + this.checkIfLoggedIn(); + } + + /** + * Opens bill search page. + */ + async openBillSearch() { + this.checkBillSearchLink(); + await this.browser.navigate('/control/szamla_search'); + } + + /** + * Submits bill search form. + */ + async submitBillSearchForm() { + this.checkIfLoggedIn(); + this.checkBillSearchForm(); + const body = 'vfw_form=szamla_search_submit&vfw_coll=szamla_search_params®szolgid=&szlaszolgid=&datumtol=&datumig='; + await this.browser.submit('/control/szamla_search_submit', body); + } + + /** + * Opens a bill. + * + * @param {string} rowId Bill's row ID + */ + async openBill(rowId) { + this.checkIfLoggedIn(); + this.checkBillSelectLink(); + await this.browser.navigate(`/control/szamla_select?vfw_coll=szamla_list&vfw_coll_index=0&vfw_rowid=${rowId}&vfw_colid=ugyfelazon|S`); + } + + /** + * Opens download page of the previously opened bill. + */ + async openBillDownloads() { + this.checkIfLoggedIn(); + this.checkBillDownloadsLink(); + await this.browser.navigate('/control/szamla_letolt'); + } + + /** + * Opens bill list, which will display the results of the previous search. + */ + async openBillList() { + this.checkIfLoggedIn(); + await this.browser.navigate('/control/szamla_list'); + } + + checkIfLoggedIn() { + this.check(this.config.user, 'Felhasználónév nem szerepel az oldalon / nem vagyunk bejelentkezve'); + this.checkBillSearchLink(); + } + + checkBillSearchLink() { + this.check('href="/ekonto/control/szamla_search"', 'Számlakereső link nem található / nem vagyunk bejelentkezve'); + } + + checkBillSearchForm() { + this.check('action="szamla_search_submit"', 'Számlakereső form nem található / nem a számlakereső oldalon vagyunk'); + } + + checkBillSelectLink() { + this.check('/control/szamla_select', 'Számla kiválasztás link nem található / nem a keresési találatok oldalán vagyunk'); + } + + checkBillDownloadsLink() { + this.check('href="szamla_letolt"', 'Számla letöltés link nem található / nem egy megnyitott számla oldalán vagyunk'); + } + + check(requiredContent, errorMessage) { + if (!this.browser.lastNavigationResponse.body.includes(requiredContent)) { + throw new Error(errorMessage); + } + } +} + +module.exports = DijnetAgent; diff --git a/src/dijnet-browser.js b/src/dijnet-browser.js new file mode 100644 index 0000000..14c621b --- /dev/null +++ b/src/dijnet-browser.js @@ -0,0 +1,126 @@ +const fs = require('fs'); +const path = require('path'); +const waitMs = require('util').promisify(setTimeout); +const mkdirp = require('mkdirp').sync; +const Browser = require('./browser'); +const Config = require('./config'); +const Logger = require('./logger'); + +/** + * Extension to `Browser`. Adds Díjnet specific base URL and HTTP headers, and + * is able to write out HTML files and cookies for further investigation. + */ +class DijnetBrowser extends Browser { + /** + * @param {Config} config Configuration + * @param {Logger} logger Logger + */ + constructor(config, logger) { + super(); + this.config = config || new Config(); + this.logger = logger || new Logger(this.config); + + this.baseUrl = 'https://www.dijnet.hu/ekonto'; + } + + /** + * Creates temporary directory if set. + * + * @returns {DijnetBrowser} This object + */ + init() { + if (this.config.tempDir) { + this.logger.verbose(`Könyvtár létrehozása: ${this.config.tempDir}`); + mkdirp(this.config.tempDir); + } + return this; + } + + /** + * Waits before sending the request, and also adds required headers and Dijnet base URL. + * + * @param {string} dijnetPath Díjnet path, relative to `baseUrl` + * @param {got.GotJSONOptions} options Request options (method, headers, body, encoding, etc.) + * @returns {got.GotPromise} Response + */ + async request(dijnetPath, options) { + this.logger.verbose(`${this.config.sleep}s várakozás...`); + await waitMs(this.config.sleep * 1000); + + this.logger.verbose(`${options.method} ${this.baseUrl}${dijnetPath}`); + const headers = Object.assign({ + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', + 'accept-language': 'hu-HU,hu;q=0.9,en-US;q=0.8,en;q=0.7' + }, options.headers || {}); + + return super.request(dijnetPath, Object.assign({ baseUrl: this.baseUrl }, options, { headers })); + } + + /** + * Sends a GET request to the given Díjnet path, then stores the response in `lastNavigationResponse`. + * If temporary directory is set, outputs the body into a HTML file as well. + * + * @param {string} dijnetPath Díjnet path, relative to `baseUrl` + * @returns {got.GotPromise} Response + */ + async navigate(dijnetPath) { + await super.navigate(dijnetPath); + this.saveTempFile(dijnetPath); + return this.lastNavigationResponse; + } + + /** + * Sends a POST request to the given Díjnet path, then stores the response in `lastNavigationResponse`. + * Also adds required headers. + * If temporary directory is set, outputs the body into a HTML file as well. + * + * @param {string} dijnetPath Díjnet path, relative to `baseUrl` + * @param {string} body Request body + * @returns {got.GotPromise} Response + */ + async submit(dijnetPath, body) { + const headers = { 'content-type': 'application/x-www-form-urlencoded' }; + await super.submit(dijnetPath, { headers, body }); + this.saveTempFile(dijnetPath); + return this.lastNavigationResponse; + } + + /** + * Downloads a file from the given Díjnet path. Filename is defined by + * Díjnet. + * + * @param {string} dijnetPath Díjnet path, relative to `baseUrl` + * @param {string} targetDir Target directory + */ + async download(dijnetPath, targetDir) { + const r = await this.request(dijnetPath, { method: 'GET', encoding: null }); + const fn = r.headers['content-disposition'].replace(/.*filename=/, ''); + const kb = Math.round(r.body.length / 102.4) / 10; + this.logger.verbose(`Fájl mentése (${kb} KB): ${fn}`); + fs.writeFileSync(path.join(targetDir, fn), r.body, 'binary'); + } + + /** + * If temporary directory is set, outputs the last response body and the + * state of the cookie jar into files. Filenames will include a timestamp + * and a normalized form of the Díjnet path. + * + * @param {string} dijnetPath Díjnet path, relative to `baseUrl` + */ + saveTempFile(dijnetPath) { + if (this.config.tempDir) { + const now = new Date(); + const ts = (now.toISOString().slice(0, 10) + ' ' + now.toLocaleTimeString()).replace(/\D/g, ''); + const nu = dijnetPath.replace(/.*dijnet.*?\//, '').replace(/[^A-Za-z0-9]+/g, '_'); + const fn = path.join(this.config.tempDir, `${ts}_${nu}.html`); + this.logger.verbose(`HTML fájl kiírása: ${fn}`); + fs.writeFileSync(fn, this.lastNavigationResponse.body); + + const fn2 = fn.replace('.html', '.cookies'); + this.logger.verbose(`Sütik kiírása: ${fn2}`); + fs.writeFileSync(fn2, JSON.stringify(this.cookieJar, null, 2)); + } + } +} + +module.exports = DijnetBrowser; diff --git a/src/dijnet-parser.js b/src/dijnet-parser.js new file mode 100644 index 0000000..ef2d2fe --- /dev/null +++ b/src/dijnet-parser.js @@ -0,0 +1,75 @@ +const cheerio = require('cheerio'); +const deburr = require('lodash.deburr'); +const Bill = require('./bill'); +const BillFile = require('./bill-file'); + +/** + * @param {string} body Response body, it should be the HTML code of a bill list page + * @returns {Bill[]} Bills' metadata + */ +function parseBillSearchResults(body) { + const $ = cheerio.load(body, { normalizeWhitespace: true }); + const thIds = $('.szamla_table th').toArray().map(th => $(th).attr('id')); + const TH_ID = { + billId: 'szl', + billIssuerId: 'aln', + dateOfIssue: 'bdt', + dueDate: 'fdt', + finalAmount: 'oss', + payable: 'egy', + serviceProvider: 'szn', + status: 'dst' + }; + + function col(id) { + const i = thIds.indexOf(id); + if (i === -1) { + throw new Error(`Nem találom a következő oszlopot: ${id} (talált oszlopok: ${thIds})`); + } + return i; + } + + function cell(tr, headerText) { + const index = col(headerText); + const text = $(tr.childNodes[index]).text(); + return text.trim(); + } + + const bills = $('.szamla_table tbody tr').toArray().map(tr => { + const bill = new Bill(); + Object.entries(TH_ID).forEach(e => { + const [field, id] = e; + bill[field] = cell(tr, id); + }); + bill.billIssuerId = normalize(bill.billIssuerId); + bill.serviceProvider = normalize(bill.serviceProvider); + bill.rowId = $(tr).html().toString().match(/rowid=(\d+)/)[1]; + return bill; + }); + return bills; +} + +/** + * @param {string} body Response body, it should be the HTML code of a bill's downloads page + * @returns {BillFile[]} Downloadable files' metadata + */ +function parseBillDownloads(body) { + const $ = cheerio.load(body, { normalizeWhitespace: true }); + return $('a.xt_link__download').toArray() + .map(a => new BillFile(normalize($(a).text()), `/control/${a.attribs.href}`)) + .filter(bf => !bf.dijnetPath.includes('://')); +} + +/** + * @param {string} s String to be normalized + * @returns {string} Normalizes input string by removing accents (á -> a), and removing non-alphanumeric characters. + */ +function normalize(s) { + return deburr(s).replace(/[^a-z0-9\-_]+/gi, ' ').trim(); +} + +module.exports = { + normalize, + parseBillSearchResults, + parseBillDownloads +}; diff --git a/src/err.js b/src/err.js deleted file mode 100644 index ed3b93a..0000000 --- a/src/err.js +++ /dev/null @@ -1,20 +0,0 @@ -const fs = require('fs'); -const chalk = require('chalk'); - -const fn = './error.log'; - -function printRed(message) { - console.log(chalk.red(message)); -} - -function handleError(error) { - if (error.message && error.stack) { - printRed(`\nHa biztos vagy abban, hogy a Díjnet felhasználóneved és jelszavad, valamint a konfigurációs paramétereket helyesen adtad meg, akkor a hiba a programban lehet.\n\nA hiba részleteit megtalálod az ${fn} fájlban. Kérlek, az alábbi linken nyiss egy új issue-t, másold be az ${fn} fájl tartalmát, és írd le röviden, milyen szituációban jelentkezett a hiba!\n\n--> https://github.com/juzraai/dijnet-bot/issues\n`); - fs.writeFileSync(fn, `${error.message}\n${error.stack}`); - } else { - printRed(error); - } - process.exit(1); -} - -module.exports = { handleError }; diff --git a/src/lib.js b/src/lib.js deleted file mode 100644 index fd2a85a..0000000 --- a/src/lib.js +++ /dev/null @@ -1,156 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const setTimeoutP = require('util').promisify(setTimeout); -const cheerio = require('cheerio'); -const deburr = require('lodash.deburr'); -const got = require('got'); -const { CookieJar } = require('tough-cookie'); -const log = require('./logger'); - -const baseUrl = 'https://www.dijnet.hu/ekonto'; -const cookieJar = new CookieJar(); - -async function _request(dijnet_path, outfile, test, body) { - log.trace('%s %s', body ? 'POST' : 'GET', dijnet_path); - const formHeaders = body ? { 'content-type': 'application/x-www-form-urlencoded' } : {}; - const headers = Object.assign({ - 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', - 'accept-language': 'hu-HU,hu;q=0.9,en-US;q=0.8,en;q=0.7' - }, formHeaders); - const options = { baseUrl, body, cookieJar, headers }; - try { - const response = await got(dijnet_path, options); - if (outfile) { - log.trace('Kapott válasz (%dKB) mentése ide: %s', Math.round(response.body.length / 102.4) / 10, outfile); - fs.writeFileSync(outfile, response.body); - } - if (test && response.body.indexOf(test) === -1) { - throw new Error(`Érvénytelen lap, nem tartalmazza ezt: ${test}`); - } - return response; - } catch (error) { - throw error; - } -} - -async function download(dijnet_path, outdir) { - log.trace('GET (binary) %s', dijnet_path); - const options = { baseUrl, cookieJar, encoding: null }; - try { - const response = await got(dijnet_path, options); - const filename = response.headers['content-disposition'].replace(/.*filename=/, ''); - const outfile = path.join(outdir, filename); - log.trace('Fájl (%d KB) mentése ide: %s', Math.round(response.body.length / 102.4) / 10, outfile); - fs.writeFileSync(outfile, response.body, 'binary'); - } catch (error) { - throw error; - } -} - -function login(dijnet_user, dijnet_pass, outfile) { - return _request( - '/login/login_check_password', outfile, - 'href="/ekonto/control/szamla_search"', - `vfw_form=login_check_password&username=${dijnet_user}&password=${dijnet_pass}`); -} - -function szamla_search(outfile) { - return _request('/control/szamla_search', outfile, 'action="szamla_search_submit"'); -} - -function szamla_search_submit(outfile) { - return _request('/control/szamla_search_submit', outfile, - '/control/szamla_select', - 'vfw_form=szamla_search_submit&vfw_coll=szamla_search_params®szolgid=&szlaszolgid=&datumtol=&datumig='); -} - -function szamla_select(rowid, outfile) { - return _request( - `/control/szamla_select?vfw_coll=szamla_list&vfw_coll_index=0&vfw_rowid=${rowid}&vfw_colid=ugyfelazon|S`, - outfile, 'href="szamla_letolt"'); -} - -function szamla_letolt(outfile) { - return _request('/control/szamla_letolt', outfile, 'class="xt_link__download"'); -} - -function szamla_list(outfile) { - return _request('/control/szamla_list', outfile, '/control/szamla_select'); -} - -function normalize(s) { - // A probléma az, hogy nagyon fura encoding-gal jönnek az ő/ű betűk, és sehogy nem tudom őket olvashatóvá konvertálni. - // Így azt találtam ki, hogy eltávolítom az ékezeteket (á->a), a nem-betű karaktereket meg kidobálom. - return deburr(s).replace(/[^a-z0-9\-_]+/gi, ' ').trim(); -} - -function parse_szamla_list(body) { - const $ = cheerio.load(body, { normalizeWhitespace: true }); - const cols = []; - $('.szamla_table th').each((_, th) => cols.push(normalize($(th).text()))); - /* - 'Szolgaltato', - 'Szamlakibocsatoi azonosito', - 'Szamlaszam', - 'Kiallitas datuma', - 'Szamla vegosszege', - 'Fizetesi hatarid', - 'Fizetend', - 'Allapot' - */ - function indexOfOrThrowError(col) { - const i = cols.indexOf(col); - if (i === -1) { - throw new Error(`Érvénytelen lap, nem tartalmazza a(z) ${col} oszlopot`); - } - return i; - } - const providerIndex = indexOfOrThrowError('Szolgaltato'); - const customNameIndex = indexOfOrThrowError('Szamlakibocsatoi azonosito'); - const dateIndex = indexOfOrThrowError('Kiallitas datuma'); - const billIdIndex = indexOfOrThrowError('Szamlaszam Bizonylatszam'); - const invoices = []; - $('.szamla_table tbody tr').each((i, tr) => { - const invoice = { - rowid: $(tr).html().toString().match(/rowid=(\d+)/)[1], - provider: normalize($(tr.childNodes[providerIndex]).text()), - customName: normalize($(tr.childNodes[customNameIndex]).text()), - date: $(tr.childNodes[dateIndex]).text(), - billId: $(tr.childNodes[billIdIndex]).text() - }; - invoices.push(invoice); - }); - return invoices; -} - -function parse_szamla_letolt(body) { - const $ = cheerio.load(body, { normalizeWhitespace: true }); - const downloads = []; - $('a.xt_link__download').each((_, a) => { - const { href } = a.attribs; - if (href.indexOf('://') === -1) { - downloads.push(`/control/${href}`); - } - }); - return downloads; -} - -async function sleep(s) { - log.trace('Várunk %d másodpercet', s); - await setTimeoutP(s * 1000); -} - -module.exports = { - // Díjnet - login, - szamla_search, - szamla_search_submit, - szamla_select, - szamla_letolt, - szamla_list, - parse_szamla_list, - parse_szamla_letolt, - download, - // util - sleep -}; diff --git a/src/logger.js b/src/logger.js index 9225e0e..bcc7686 100644 --- a/src/logger.js +++ b/src/logger.js @@ -1,65 +1,123 @@ -const chalk = require('chalk'); -const { Signale } = require('signale'); +const fs = require('fs'); +const kleur = require('kleur'); +const packageInfo = require('../package.json'); +const Config = require('./config'); -const config = { - coloredInterpolation: false, - displayScope: false, - displayBadge: true, - displayDate: false, - displayFilename: false, - displayLabel: false, - displayTimestamp: true, - underlineLabel: true, - underlineMessage: false, - underlinePrefix: false, - underlineSuffix: false, - uppercaseLabel: false -}; +function getTimestamp() { + const now = new Date(); + return now.toISOString().slice(0, 10) + ' ' + now.toLocaleTimeString(); +} -const options = { - types: { - trace: { - badge: '…', - color: 'gray', - label: '' - }, - info: { - badge: '·', - color: 'blue', - label: '' - }, - warn: { - color: 'yellow', - label: '' - }, - error: { - color: 'red', - label: '' - }, - success: { - color: 'green', - label: '' +/** + * Logger with quiet and verbose mode, and 4 message types. In non-interactive + * standard outputs (e.g. redirects), verbose mode will be activated. Verbose + * mode prints out every log message with a timestamp prefix. + */ +class Logger { + /** + * @param {Config} config Configuration + */ + constructor(config) { + this.config = config || new Config(); + } + + /** + * Prints out application version. + * + * @returns {Logger} This object + */ + init() { + this.log(`DíjnetBot v${packageInfo.version}\n`, kleur.white, true); + return this; + } + + /** + * Prints out the given message to standard output. + * + * @param {string} message Message + * @param {kleur.Color} colorFunc Kleur color function + * @param {boolean} bold Whether message should be displayed with bold font + */ + log(message, colorFunc, bold) { + if (this.config.quiet) { + return; + } + + let s = message; + + if (this.config.verbose) { + s = s.trim() + .split('\n') + .filter(line => line.trim().length > 0) + .join('\n'); } + + if (this.config.tty) { + if (bold) { + s = kleur.bold(s); + } + if (colorFunc) { + s = colorFunc(s); + } + } + + if (this.config.verbose || !this.config.tty) { + const ts = getTimestamp(); + console.log(s.split('\n').map(line => kleur.reset().gray(`${ts} | `) + line).join('\n')); + return; + } + + console.log(s); } -}; -const log = new Signale(options); -log.config(config); -Object.keys(options.types).forEach(t => { - const original = log[t]; - log[t] = (...s) => { - const { color } = options.types[t]; - s[0] = chalk[color](s[0]); - original(...s); - }; -}); + /** + * Prints out an error message in red. + * + * @param {(string|Error)} error Error message or object + */ + error(error) { + this.log(error.message || error, kleur.red, true); + if (error.stack) { + const s = '\nHa biztos vagy abban, hogy helyesen konfiguráltad a Díjnet Bot-ot, akkor a hiba a programban lehet.\nKérlek, az alábbi linken nyiss egy új issue-t, másold be a hiba részleteit, és írd le röviden, milyen szituációban jelentkezett a hiba!\n\n--> https://github.com/juzraai/dijnet-bot/issues\n\n'; + this.log(s, kleur.red, true); + try { + fs.writeFileSync(this.config.errorFile, `${getTimestamp()}: ${error.message}\n${error.stack}`); + this.log(`A hiba részleteit megtalálod a(z) ${this.config.errorFile} fájlban.`, kleur.red, true); + } catch (_) { + this.log('A hiba részletei:', kleur.red, true); + this.log(error.stack, kleur.red, false, false); + } + } + } -const types = { trace: 4, info: 3, success: 2, error: 1 }; // ezeket használjuk csak -Object.keys(types).forEach(t => { - const l = types[t]; - if (process.env.LOG_LEVEL < l) { - log[t] = () => {}; + /** + * Prints out a success message in green. + * + * @param {string} message Message indicating success + */ + success(message) { + this.log(message, kleur.green, true); + } + + /** + * Prints out an info message. + * + * @param {string} message Message informing about an operation in progress + */ + info(message) { + this.log(message, null, false); + } + + /** + * Prints out a verbose-level (trace) message in grey. + * + * @param {string} message Message describing a detail of a process + */ + verbose(message) { + if (this.config.verbose) { + this.log(message, kleur.grey, false); + } } -}); +} -module.exports = log; +module.exports = Logger; diff --git a/src/main.js b/src/main.js index 5392126..2eda3ef 100644 --- a/src/main.js +++ b/src/main.js @@ -1,103 +1,53 @@ -const fs = require('fs'); -const path = require('path'); -const mkdirp = require('util').promisify(require('mkdirp')); -const packageJson = require('../package.json'); -const configurate = require('./conf'); -const { handleError } = require('./err'); - -console.log(`Díjnet Bot v${packageJson.version}\n`); - -configurate(); -const log = require('./logger'); -const dijnet = require('./lib'); - -function tmp(name) { - return process.env.TEMP_DIR.length === 0 ? null : path.join(process.env.TEMP_DIR, name); -} - -const alreadyCrawledIdsFile = path.join(process.env.OUTPUT_DIR, 'kesz.txt'); -let alreadyCrawledIds = null; -function isAlreadyCrawled(id) { - if (alreadyCrawledIds === null) { - if (fs.existsSync(alreadyCrawledIdsFile)) { - alreadyCrawledIds = fs.readFileSync(alreadyCrawledIdsFile, 'utf8').split('\n'); - } else { - alreadyCrawledIds = []; - } - } - return alreadyCrawledIds.includes(id); -} - -function markAlreadyCrawled(id) { - fs.appendFileSync(alreadyCrawledIdsFile, id + '\n'); -} - -const start = async () => { - try { - log.success('Díjnet-bot indul'); - - log.trace('Könyvtárak létrehozása'); - log.trace('Kimeneti könyvtár létrehozása: %s', process.env.OUTPUT_DIR); - await mkdirp(process.env.OUTPUT_DIR); - if (process.env.TEMP_DIR.length > 0) { - log.trace('Segédfájlok könyvtár létrehozása: %s', process.env.TEMP_DIR); - await mkdirp(process.env.TEMP_DIR); +const configurator = require('./configurator'); +const DijnetAgent = require('./dijnet-agent'); +const DijnetBrowser = require('./dijnet-browser'); +const parser = require('./dijnet-parser'); +const Logger = require('./logger'); +const Repo = require('./repo'); + +/** + * Díjnet Bot itself. Loads configuration, initializes components, logs in to + * Díjnet, searches for bills, iterates over them and downloads every file it + * founds. Maintains a list of previously completed bills and skips them. + */ +async function start() { + const config = await configurator.getConfig(); + const logger = new Logger(config).init(); + const repo = new Repo(config, logger).init(); + const browser = new DijnetBrowser(config, logger).init(); + const agent = new DijnetAgent(config, logger, browser); + + logger.info('Bejelentkezés...'); + await agent.login(); + logger.success(`Bejelentkezve: ${config.user}`); + + logger.info('Számlák keresése...'); + await agent.openBillSearch(); + await agent.submitBillSearchForm(); + let bills = parser.parseBillSearchResults(agent.browser.lastNavigationResponse.body); + const allBillsCount = bills.length; + bills = bills.filter(repo.isNew.bind(repo)); + logger.success(`${allBillsCount} db számla van a rendszerben: ${bills.length} db új, ${allBillsCount - bills.length} db lementve korábban`); + + for (let i = 0; i < bills.length; i++) { + const bill = bills[i]; + const prefix = `${i}/${bills.length} db új számla lementve (${Math.round(i / bills.length * 100)}%) | ${bill.dateOfIssue} ${bill.serviceProvider}`; + logger.info(`${prefix} | Megnyitás...`); + await agent.openBill(bill.rowId); + await agent.openBillDownloads(); + + const files = parser.parseBillDownloads(agent.browser.lastNavigationResponse.body); + for (let j = 0; j < files.length; j++) { + const file = files[j]; + logger.info(`${prefix} | [${j + 1}/${files.length}] ${file.name}`); + await browser.download(file.dijnetPath, repo.directoryFor(bill)); } + repo.markAsDone(bill); - log.info('Bejelentkezés...'); - await dijnet.login(process.env.DIJNET_USER, process.env.DIJNET_PASS, tmp('login.html')); - log.success(`Bejelentkezve: ${process.env.DIJNET_USER}`); - - log.info('Számlák keresése...'); - await dijnet.sleep(process.env.SLEEP); - await dijnet.szamla_search(tmp('szamla_search.html')); - await dijnet.sleep(process.env.SLEEP); - const szamla_list_response = (await dijnet.szamla_search_submit(tmp('szamla_search_submit.html'))).body; - - let invoices = dijnet.parse_szamla_list(szamla_list_response); - const allBillsCount = invoices.length; - invoices = invoices.filter(invoice => !isAlreadyCrawled(invoice.billId)); - log.success(`${allBillsCount} db számla van a rendszerben, ebből ${allBillsCount - invoices.length} db lementve korábban`); - - for (let i = 0; i < invoices.length; i++) { - const invoice = invoices[i]; - const dir = path.join(process.env.OUTPUT_DIR, `${invoice.provider} - ${invoice.customName}`, invoice.date); - - log.info(`Számla lementése: ${invoice.date}, ${invoice.provider}`); - - const logPrefix = `Számla ${i + 1}/${invoices.length} (${invoice.date}, ${invoice.provider}):`; - - log.trace(`${logPrefix} megnyitás`); - await mkdirp(dir); - await dijnet.sleep(process.env.SLEEP); - await dijnet.szamla_select(invoice.rowid, tmp(`szamla_select_${invoice.rowid}.html`)); - - await dijnet.sleep(process.env.SLEEP); - const szamla_letolt_response = (await dijnet.szamla_letolt(tmp(`szamla_letolt_${invoice.rowid}.html`))).body; - - const files = dijnet.parse_szamla_letolt(szamla_letolt_response); - for (let f = 0; f < files.length; f++) { - const file = files[f]; - log.trace(`${logPrefix} ${file} letöltése`); - await dijnet.sleep(process.env.SLEEP); - await dijnet.download(file, dir); - } - - markAlreadyCrawled(invoice.billId); - log.trace(`${logPrefix} ${files.length} fájl lementve`); - log.success(`${invoices.length} db új számlából ${i + 1} db lementve [${Math.round((i + 1) / invoices.length * 100)}%]`); - log.trace('Visszatérés a számla listához'); - await dijnet.sleep(3); - await dijnet.szamla_list(tmp(`szamla_list_${invoice.rowid}.html`)); - } - log.success('Kész'); - - process.exit(0); // Windows 10-en Git Bash-ben különben nem áll le a program valamiért - } catch (error) { - log.trace(error.stack); - log.error(error.message); - handleError(error); + await agent.openBillList(); } -}; + logger.success(`${bills.length} db új számla lementve!`); + process.exit(0); +} module.exports = { start }; diff --git a/src/repo.js b/src/repo.js new file mode 100644 index 0000000..a550c3d --- /dev/null +++ b/src/repo.js @@ -0,0 +1,80 @@ +const fs = require('fs'); +const path = require('path'); +const mkdirp = require('mkdirp').sync; +const Bill = require('./bill'); // eslint-disable-line no-unused-vars +const Config = require('./config'); +const Logger = require('./logger'); + +/** + * Utility for output directory management and maintaining completed bills. + */ +class Repo { + /** + * @param {Config} config Configuration + * @param {Logger} logger Logger + */ + constructor(config, logger) { + this.config = config || new Config(); + this.logger = logger || new Logger(this.config); + + this.crawledBillIds = []; + this.doneFile = path.join(this.config.outputDir, this.config.doneFile); + } + + /** + * Creates output directory, then loads bill IDs from done-file. + * + * @returns {Repo} This object + */ + init() { + this.logger.verbose(`Könyvtár létrehozása: ${this.config.outputDir}`); + mkdirp(this.config.outputDir); + + this.logger.verbose(`Kész-lista beolvasása: ${this.doneFile}`); + if (fs.existsSync(this.doneFile)) { + this.crawledBillIds = fs.readFileSync(this.doneFile, 'utf8').split('\n'); + } + this.logger.verbose(`${this.crawledBillIds.length} db számla van a kimeneti mappában`); + + return this; + } + + /** + * Creates new directory for given bill, returns directory path. + * + * @param {Bill} bill Bill + * @returns {string} Download directory for given bill + */ + directoryFor(bill) { + const d = path.join(this.config.outputDir, `${bill.serviceProvider} - ${bill.billIssuerId}`, bill.dateOfIssue); + mkdirp(d); + return d; + } + + /** + * @param {Bill} bill Bill + * @returns {boolean} Whether bill is already downloaded completely (false means it is new) + */ + isDone(bill) { + return this.crawledBillIds.includes(bill.billId); + } + + /** + * @param {Bill} bill Bill + * @returns {boolean} Whether bill is new (false means it is already downloaded) + */ + isNew(bill) { + return !this.isDone(bill); + } + + /** + * Marks given bill as downloaded completely. + * + * @param {Bill} bill Bill + */ + markAsDone(bill) { + fs.appendFileSync(this.doneFile, `${bill.billId}\n`); + } +} + +module.exports = Repo;