Merge pull request #10 from code-yeongyu/naver-parser

`NaverPay` 의 구매기록을 조회 할 경우 `pupeteer` 대신 `axios` 를 활용해 `http request` 를 보내어 parsing 하도록 변경합니다.

example/naver/printPaymentHistory.ts

@@ -18,8 +18,9 @@ const printNaverPayHistory = async (id: string, password: string) => {
   const module = NaverApp.ModuleFactory.create(page);
   const crawlService = new NaverApp.Service(module);
 
-  await crawlService.normalLogin(id, password);
+  await crawlService.normalLogin(id, password, 100);
 
+  browser.close();
   const history = await crawlService.getHistory();
   console.log(history);
 };

example/naver/reactivelyPrintPaymentHistory.ts

@@ -1,9 +1,9 @@
 import puppeteer from "puppeteer";
-import { NaverApp } from ".";
+import { NaverApp } from "trackpurchase";
 
 import readline from "readline";
 import { concat, defer, filter, from, tap } from "rxjs";
-import { CaptchaStatus } from "app/naver";
+import { CaptchaStatus } from "trackpurchase/app/naver";
 
 const printNaverPayHistory = async (id: string, password: string) => {
   const MOBILE_UA =
@@ -21,7 +21,7 @@ const printNaverPayHistory = async (id: string, password: string) => {
   const module = NaverApp.ModuleFactory.create(page);
   const crawlService = new NaverApp.Service(module);
 
-  const loginEvent$ = crawlService.interactiveLogin(id, password);
+  const loginEvent$ = crawlService.interactiveLogin(id, password, 100);
   const history$ = defer(() => from(crawlService.getHistory()));
   const closePage$ = defer(() => from(page.close()));
   const closeBrowser$ = defer(() => from(browser.close()));

package.json

@@ -1,6 +1,6 @@
 {
   "name": "trackpurchase",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "main": "dist/index.js",
   "license": "MIT",
   "repository": {
@@ -34,6 +34,7 @@
     "build": "tsc -p ."
   },
   "dependencies": {
+    "axios": "^0.25.0",
     "puppeteer": "^12.0.1",
     "rxjs": "^7.5.2"
   },
@@ -43,6 +44,7 @@
     "@babel/preset-env": "^7.16.5",
     "@babel/preset-typescript": "^7.16.5",
     "@babel/runtime": "^7.16.5",
+    "@types/axios": "^0.14.0",
     "@types/expect-puppeteer": "^4.4.7",
     "@types/jest": "^27.0.3",
     "@types/jest-environment-puppeteer": "^4.4.1",

src/app/common/index.ts

@@ -1,4 +1,4 @@
 import Module from "./module";
-import PaymentHistory from "./paymentHistory";
+import PaymentHistory from "./types/paymentHistory";
 
 export { Module, PaymentHistory };

src/app/common/types/response.d.ts

@@ -0,0 +1,4 @@
+export interface CommonResponse {
+  readonly status: number;
+  readonly data: string;
+}

src/app/naver/index.ts

@@ -4,6 +4,8 @@ import URLChanger from "./urlChanger";
 import PageInteractor, { LoginEvent, CaptchaStatus } from "./pageInteractor";
 import ElementParser from "./elementParser";
 import Service from "./service";
+import { NaverScraper } from "./scraper";
+import { NaverParser } from "./parser";
 
 export {
   Module,
@@ -14,4 +16,6 @@ export {
   Service,
   LoginEvent,
   CaptchaStatus,
+  NaverScraper,
+  NaverParser,
 };

src/app/naver/module.ts

@@ -1,8 +1,16 @@
-import { URLChanger, PageInteractor, ElementParser } from ".";
+import {
+  URLChanger,
+  PageInteractor,
+  ElementParser,
+  NaverScraper,
+  NaverParser,
+} from ".";
 import { Module as BaseModule } from "../common";
 
 export default interface Module extends BaseModule {
   readonly urlChanger: URLChanger;
   readonly pageInteractor: PageInteractor;
   readonly elementParser: ElementParser;
+  readonly scraper: NaverScraper;
+  readonly parser: NaverParser;
 }

src/app/naver/moduleFactory.ts

@@ -1,16 +1,27 @@
 import puppeteer from "puppeteer";
-import { Module, URLChanger, ElementParser, PageInteractor } from ".";
+import {
+  Module,
+  URLChanger,
+  ElementParser,
+  PageInteractor,
+  NaverScraper,
+  NaverParser,
+} from ".";
 
 export default class ModuleFactory {
   static create(page: puppeteer.Page): Module {
     const urlChanger = new URLChanger(page);
     const elementParser = new ElementParser(page);
     const pageInteractor = new PageInteractor(page, elementParser);
+    const scraper = new NaverScraper();
+    const parser = new NaverParser();
 
     return {
       urlChanger,
       pageInteractor,
       elementParser,
+      scraper,
+      parser,
     };
   }
 }

src/app/naver/pageInteractor.ts

@@ -50,6 +50,15 @@ export default class PageInteractor {
     await this.typeLoginInfo(id, password, delay || 200);
   }
 
+  async getCookies() {
+    const cookies = await this.page.cookies();
+    let cookieString = "";
+    for (const cookie of cookies) {
+      cookieString += `${cookie.name}=${cookie.value}; `;
+    }
+    return cookieString;
+  }
+
   async getLoginStatus(loginURL?: string): Promise<LoginEvent> {
     const isLoginPage = this.page
       .url()

src/app/naver/parser.spec.ts

@@ -0,0 +1,37 @@
+import { NaverParser } from "./parser";
+import {
+  expectedPaymentHistoryItems,
+  searchPaymentHistoryJson,
+} from "./testFixture";
+
+describe("NaverParser", () => {
+  describe("parsePaymentHistory", () => {
+    it("should parse", () => {
+      // given
+      const parser = new NaverParser();
+
+      // when
+      const result = parser.parsePaymentHistory(searchPaymentHistoryJson);
+
+      // then
+      expect(result).toEqual(expectedPaymentHistoryItems);
+    });
+  });
+
+  describe("parseInformationForNextPaymentHistory", () => {
+    it("should parse", () => {
+      // given
+      const parser = new NaverParser();
+
+      // when
+      const result = parser.parseInformationForNextPaymentHistory(
+        searchPaymentHistoryJson
+      );
+
+      // then
+      expect(result.hasNext).toBe(true);
+      expect(result.lastHistoryId).toBe("order-2021110243152861");
+      expect(result.lastHistoryDateTimestamp).toBe(1635853254000);
+    });
+  });
+});

src/app/naver/parser.ts

@@ -0,0 +1,40 @@
+import { PaymentHistory } from "app/common";
+import { PaymentHistoryResponse } from "./types/paymentHistoryResponse";
+
+export class NaverParser {
+  parsePaymentHistory(jsonString: string): PaymentHistory[] {
+    const data = JSON.parse(jsonString) as PaymentHistoryResponse;
+    const items = data.result.items;
+    const paymentHistories = items.map((item): PaymentHistory => {
+      const name = item.product.name;
+      const price = item.product.price;
+      const thumbnailURL = item.product.imgUrl;
+      const paymentStatus = item.status.text;
+      const isAdditional = !!item.additionalData.isSupplemented;
+      const purchasedAtTimestamp = item.date;
+      const purchasedAt = new Date(purchasedAtTimestamp);
+      return {
+        name,
+        price,
+        thumbnailURL,
+        paymentStatus,
+        isAdditional,
+        purchasedAt,
+      };
+    });
+    return paymentHistories;
+  }
+
+  parseInformationForNextPaymentHistory(jsonString: string): {
+    hasNext: boolean;
+    lastHistoryId: string;
+    lastHistoryDateTimestamp: number;
+  } {
+    const data = JSON.parse(jsonString) as PaymentHistoryResponse;
+
+    const hasNext = data.result.hasNext;
+    const lastHistoryId = data.result.lastId;
+    const lastHistoryDateTimestamp = data.result.lastDate;
+    return { hasNext, lastHistoryId, lastHistoryDateTimestamp };
+  }
+}

src/app/naver/scraper.test.ts

@@ -0,0 +1,60 @@
+import axios from "axios";
+import { NaverScraper } from "./scraper";
+describe("Scraper", () => {
+  describe("searchPaymentHistory", () => {
+    it("should create an post request", async () => {
+      // given
+      const cookies = "";
+      const scraper = new NaverScraper();
+      const postSpy = jest.spyOn(axios, "post");
+      postSpy.mockImplementation(() => Promise.resolve({ data: {} }));
+
+      // when
+      await scraper.searchPaymentHistory(cookies);
+
+      // then
+      expect(postSpy).toBeCalledWith(
+        scraper.searchPaymentHistoryURL,
+        expect.any(Object),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            Cookie: cookies,
+          }),
+        })
+      );
+    });
+  });
+
+  describe("nextPaymentHistory", () => {
+    it("should create an post request", async () => {
+      // given
+      const second = 1000;
+      const minute = second * 60;
+      const hour = minute * 60;
+      const day = hour * 24;
+
+      const cookies = "";
+      const scraper = new NaverScraper();
+      const postSpy = jest.spyOn(axios, "post");
+      postSpy.mockImplementation(() => Promise.resolve({ data: {} }));
+
+      // when
+      await scraper.nextPaymentHistory(
+        cookies,
+        "order-1234",
+        new Date().getTime() - day * 30
+      );
+
+      // then
+      expect(postSpy).toBeCalledWith(
+        scraper.nextPaymentHistoryURL,
+        expect.any(Object),
+        expect.objectContaining({
+          headers: expect.objectContaining({
+            Cookie: cookies,
+          }),
+        })
+      );
+    });
+  });
+});

src/app/naver/scraper.ts

@@ -0,0 +1,83 @@
+import { CommonResponse } from "app/common/types/response";
+import axios from "axios";
+import { ServiceGroup } from "./types/serviceGroup";
+import { StatusGroup } from "./types/statusGroup";
+
+export class NaverScraper {
+  private async getTodayString() {
+    const today = new Date();
+    const year = today.getFullYear();
+    const month = today.getMonth() + 1;
+    const date = today.getDate();
+    const monthString = month >= 10 ? month : "0" + month;
+    const dateString = date >= 10 ? date : "0" + date;
+    return `${year}-${monthString}-${dateString}`;
+  }
+
+  searchPaymentHistoryURL =
+    "https://new-m.pay.naver.com/api/timeline/searchPaymentHistory";
+  async searchPaymentHistory(
+    cookies: string,
+    searchOptions?: {
+      keyword: string;
+      serviceGroup: ServiceGroup;
+      statusGroup: StatusGroup;
+    }
+  ): Promise<CommonResponse> {
+    const data = {
+      keyword: searchOptions?.keyword || null,
+      startDate: "2000-01-01",
+      endDate: await this.getTodayString(),
+      serviceGroup: searchOptions?.serviceGroup || null,
+      statusGroup: searchOptions?.statusGroup || null,
+    };
+    const options = {
+      headers: {
+        Cookie: cookies,
+        "content-type": "application/json;charset=UTF-8",
+      },
+    };
+    const response = await axios.post(
+      this.searchPaymentHistoryURL,
+      data,
+      options
+    );
+
+    return { status: response.status, data: JSON.stringify(response.data) };
+  }
+
+  nextPaymentHistoryURL =
+    "https://new-m.pay.naver.com/api/timeline/nextPaymentHistory";
+  async nextPaymentHistory(
+    cookies: string,
+    lastHistoryId: string,
+    lastHistoryDateTimestamp: number,
+    searchOptions?: {
+      keyword: string;
+      serviceGroup: ServiceGroup;
+      statusGroup: StatusGroup;
+    }
+  ): Promise<CommonResponse> {
+    const data = {
+      keyword: searchOptions?.keyword || null,
+      startDate: "2000-01-01",
+      endDate: await this.getTodayString(),
+      serviceGroup: searchOptions?.serviceGroup || null,
+      statusGroup: searchOptions?.statusGroup || null,
+      lastId: lastHistoryId,
+      lastDate: lastHistoryDateTimestamp,
+    };
+    const options = {
+      headers: {
+        Cookie: cookies,
+        "content-type": "application/json;charset=UTF-8",
+      },
+    };
+    const response = await axios.post(
+      this.nextPaymentHistoryURL,
+      data,
+      options
+    );
+    return { status: response.status, data: JSON.stringify(response.data) };
+  }
+}