Skip to content

Commit

Permalink
๐Ÿ”จ Fix: ์‹œ๊ฐ„์ œํ•œ 1ํŠธ 3๋ถ„์œผ๋กœ ์žฌ๋ณ€๊ฒฝ
Browse files Browse the repository at this point in the history
#
  • Loading branch information
qilip committed Nov 14, 2023
1 parent d2388ce commit bcc40a1
Showing 1 changed file with 4 additions and 33 deletions.
37 changes: 4 additions & 33 deletions src/scrap/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,39 +74,10 @@ const singleFile = async (args: object, url: string) => {
};

const scrapeJob = async (args: any, url: string) => {
try {
// ์Šคํฌ๋žฉ 1์ฐจ ์‹œ๋„ 1๋ถ„ 30์ดˆ
console.log('scrape 1st try start');
await promiseTimeout(90 * 1000, singleFile(args, url));
if (!fs.existsSync(args.output)) {
console.log('scrape 1st failed. try 2nd');
throw new Error('HTML file not found. scrap failed');
}
} catch (e) {
// ์Šคํฌ๋žฉ 2์ฐจ ์‹œ๋„ 3๋ถ„, ๋™์‹œ์— 3๊ฐœ ์‹œ๋„ํ•ด์„œ 1๊ฐœ๋ผ๋„ ์„ฑ๊ณตํ•˜๋ฉด ๋
// settimeout์œผ๋กœ 10์ดˆ์”ฉ ์ง€์—ฐ ๋‘๊ณ  3๊ฐœ ๋™์‹œ์— ์‹œ๋„
const delayStart = (timeout, taskFunc) => {
return new Promise((resolve, reject) => {
setTimeout(() => {
taskFunc().then(resolve).catch(reject);
}, timeout);
});
};
console.log('scrape 2nd try start');
await Promise.race([
promiseTimeout(180 * 1000, singleFile(args, url)),
delayStart(10 * 1000, () =>
promiseTimeout(170 * 1000, singleFile(args, url)),
),
delayStart(20 * 1000, () =>
promiseTimeout(160 * 1000, singleFile(args, url)),
),
]);

if (!fs.existsSync(args.output)) {
console.log('scrape 2nd failed.');
throw new Error('HTML file not found. scrap failed');
}
console.log('scrape start');
await promiseTimeout(180 * 1000, singleFile(args, url));
if (!fs.existsSync(args.output)) {
throw new Error('HTML file not found. scrap failed');
}
};

Expand Down

0 comments on commit bcc40a1

Please sign in to comment.