feat(apis): init mapping in apis ✨

lib/helpers/montaineMapping.js

@@ -0,0 +1,43 @@
+/**
+ * Module dependencies
+ */
+const path = require('path');
+const _ = require('lodash');
+
+const AppError = require(path.resolve('./lib/helpers/AppError'));
+
+const objectDeepKeys = (obj) => Object.keys(obj).filter((key) => obj[key] instanceof Object).map((key) => objectDeepKeys(obj[key]).map((k) => `${key}.${k}`)).reduce((x, y) => x.concat(y), Object.keys(obj));
+
+const prepareFormat = (object, schema) => {
+  const result = {};
+  const keys = objectDeepKeys(schema);
+  keys.forEach((k) => {
+    const p = k.split('.');
+    if (!(/^\d+$/.test(p[p.length - 1])) && !Array.isArray(_.get(schema, k))) {
+      const targetPath = _.get(schema, k);
+      const targetValue = _.get(object, targetPath);
+      if (!targetValue) throw new AppError('Mapping : one of your path value is wrong.', { code: 'HELPERS_ERROR' });
+      if (!Array.isArray(targetValue) && targetValue) {
+        _.set(result, k, targetValue);
+      } else {
+        targetValue.forEach((v, i) => {
+          const generatePath = p;
+          generatePath[generatePath.length - 2] = i;
+          _.set(result, generatePath.join('.'), v);
+        });
+        _.set(result, k, targetValue[0]);
+      }
+    }
+  });
+  return result;
+};
+
+
+exports.mapping = (json, schema) => {
+  if (Array.isArray(json) && Array.isArray(schema)) {
+    return json.map((j) => prepareFormat(j, schema[0]));
+  } if (typeof example === 'object' && typeof data === 'object') {
+    return prepareFormat(json, schema);
+  }
+  throw new AppError('Typing & scrap data return aren\'t arrays or objects', { code: 'HELPERS_ERROR' });
+};

lib/helpers/montaine.js → lib/helpers/montaineRequest.js

@@ -2,36 +2,56 @@
  * Module dependencies
  */
 const axios = require('axios');
+const moment = require('moment');
 const jwt = require('jsonwebtoken');
 const path = require('path');
 
 const config = require(path.resolve('./config'));
+const AppError = require(path.resolve('./lib/helpers/AppError'));
 
 
+/**
+ * @desc generate default params
+ * @param {Object} p - params
+ * @return {Object} result - params generated
+ */
+exports.setParams = (p) => {
+  const result = {};
+  if (p && p.length !== 0) {
+    Object.keys(p).forEach((key) => {
+      if (String(p[key]).split('@')[0] === 'Date') result[key] = moment().add(Number(String(p[key]).split('@')[1]), 'days').format(String(p[key]).split('@')[2]);
+      else result[key] = p[key];
+    });
+  }
+  return result;
+};
+
 /**
  * @desc request
  * @param {String} r - request
  * @return {} result
  */
-exports.request = async (api) => {
+exports.request = async (api, params) => {
   try {
     const token = jwt.sign({ userId: api.serviceId }, config.jwtLou.secret, { expiresIn: config.jwtLou.expiresIn });
 
     const res = await axios({
       method: 'POST',
-      url: 'http://localhost:3010/api/scraps/worker/5e567f72778ecf488ce269f9',
+      url: api.url,
       headers: {
         Cookie: `TOKEN=${token}`,
       },
-      data: {},
+      data: params,
     });
+    if (api.auth === 'lou' && res.data) return res.data;
     return res;
   } catch (err) {
-    if (err.response && err.response.data) return err.response.data;
-    return { data: 'distant server not reachable' };
+    if (err.response && err.response.data) return err.response;
+    throw new AppError(`Distant server (${api.url}) not reachable.`, { code: 'HELPERS_ERROR' });
   }
 };
 
+
 /**
  * @desc setScrapHistory
  * @param {Object} data - scraping result

lib/helpers/montaineTyping.js

@@ -0,0 +1,80 @@
+/**
+ * Module dependencies
+ */
+const moment = require('moment');
+const path = require('path');
+const _ = require('lodash');
+
+const AppError = require(path.resolve('./lib/helpers/AppError'));
+
+
+/**
+ * @desc t
+ * @param {Object} p - params
+ * @return {Object} result - params generated
+ */
+const objectDeepKeys = (obj) => Object.keys(obj).filter((key) => obj[key] instanceof Object).map((key) => objectDeepKeys(obj[key]).map((k) => `${key}.${k}`)).reduce((x, y) => x.concat(y), Object.keys(obj));
+
+const format = (data, action, object) => {
+  const func = action.split('(')[0];
+  const param = action.split('(')[1].slice(0, -1);
+
+  switch (func) {
+    case 'DATE':
+      try {
+        return moment(new Date(data)).format();
+      } catch (err) {
+        throw new AppError('Typing : format DATE error', { code: 'HELPERS_ERROR' });
+      }
+    case 'NUMBER':
+      try {
+        data = String(data).replace(/,/g, '.'); // switch , to .
+        data = String(data).replace(/[^\d.-]/g, ''); // remove if it's not letters
+        return Number(data);
+      } catch (err) {
+        throw new AppError('Typing : format NUMBER error', { code: 'HELPERS_ERROR' });
+      }
+    case 'STRING':
+      try {
+        return String(data);
+      } catch (err) {
+        throw new AppError('Typing : format STRING error', { code: 'HELPERS_ERROR' });
+      }
+    case 'HOUR':
+      try {
+        data = String(data).replace(/h/g, ':');
+        data = String(data).replace(/s/g, ':');
+        if (param) data = `${_.get(object, param)} ${data}`;
+        return moment(new Date(data)).format();
+      } catch (err) {
+        throw new AppError('Typing : format HOURS error', { code: 'HELPERS_ERROR' });
+      }
+    default:
+      throw new AppError('Typing : format unknown', { code: 'HELPERS_ERROR' });
+  }
+};
+
+const prepareFormat = (object, schema) => {
+  const result = {};
+  const keys = objectDeepKeys(object); // get all keys
+  keys.forEach((k) => {
+    if (!Array.isArray(_.get(object, k))) {
+      const p = k.split('.');
+      let func = '';
+      if (/^\d+$/.test(p[p.length - 1])) func = _.get(schema, p.slice(0, -1).join('.'))[0];
+      else func = _.get(schema, p.join('.'));
+      _.set(result, k, format(_.get(object, k), func, object));
+    }
+  });
+  return result;
+};
+
+
+exports.typing = (json, schema) => {
+  if (Array.isArray(json) && Array.isArray(schema)) {
+    return json.map((j) => prepareFormat(j, schema[0]));
+  } if (typeof example === 'object' && typeof data === 'object') {
+    return prepareFormat(json, schema);
+  }
+  throw new AppError('Typing : Typing & scrap data return aren\'t arrays or objects', { code: 'HELPERS_ERROR' });
+};

modules/apis/models/apis.model.mongoose.js

@@ -13,6 +13,9 @@ const ApiMongoose = new Schema({
   url: String,
   auth: String,
   serviceId: String,
+  params: {},
+  typing: String,
+  mapping: String,
   status: Boolean,
   banner: String,
   description: String,

modules/apis/models/apis.schema.js

@@ -15,6 +15,9 @@ const ApiSchema = Joi.object().keys({
   url: Joi.string().trim().required(),
   auth: Joi.string().valid(['lou']).required(),
   serviceId: Joi.string().trim().default('').required(),
+  params: Joi.object({}).unknown().optional(),
+  typing: Joi.string().trim().allow(null).optional(),
+  mapping: Joi.string().trim().allow(null).optional(),
   status: Joi.boolean().default(false).optional(),
   banner: Joi.string().trim().default('').allow('')
     .optional(),

modules/apis/services/apis.service.js

@@ -4,7 +4,9 @@
 const path = require('path');
 
 const UserService = require(path.resolve('./modules/users/services/user.service.js'));
-const montaine = require(path.resolve('./lib/helpers/montaine'));
+const montaineMapping = require(path.resolve('./lib/helpers/montaineMapping'));
+const montaineTyping = require(path.resolve('./lib/helpers/montaineTyping'));
+const montaineRequest = require(path.resolve('./lib/helpers/montaineRequest'));
 const ApisRepository = require('../repositories/apis.repository');
 const HistoryRepository = require('../repositories/history.repository');
 
@@ -51,10 +53,14 @@ exports.update = async (api, body) => {
   api.url = body.url;
   api.auth = body.auth;
   api.serviceId = body.serviceId;
+  api.params = body.params;
   api.status = body.status;
   api.banner = body.banner;
   api.description = body.description;
-  api.password = await UserService.hashPassword(api.password);
+  if (body.typing && body.typing !== '') api.typing = body.typing;
+  else api.typing = null;
+  if (body.mapping && body.mapping !== '') api.mapping = body.mapping;
+  else api.mapping = null;
 
   const result = await ApisRepository.update(api);
   return Promise.resolve(result);
@@ -78,25 +84,31 @@ exports.delete = async (api) => {
  */
 exports.load = async (api) => {
   const start = new Date();
+  const result = {};
+  // conf
+  const params = montaineRequest.setParams(api.params);
 
-  let result = await montaine.request(api);
-  result = result.data;
+  const request = await montaineRequest.request(api, params);
+  result.request = request;
 
-  const history = await HistoryRepository.create(montaine.setScrapHistory(result, api, start));
+  // test data
+  if (result.request.data && result.request.type === 'success' && api.typing && api.typing !== '') {
+    result.typing = montaineTyping.typing(result.request.data, JSON.parse(api.typing));
+  }
+
+  // test data
+  if (result.typing && api.mapping && api.mapping !== '') {
+    result.mapping = montaineMapping.mapping(result.typing, JSON.parse(api.mapping));
+  }
+
+
+  const history = await HistoryRepository.create(montaineRequest.setScrapHistory(result, api, start));
   api.status = result.type === 'success';
   api.history.push(history._id);
   api = await ApisRepository.update(api);
   // return
   return Promise.resolve({
-    result,
     api,
+    result,
   });
 };
-
-
-// const historySchema = Joi.object().keys({
-//   apiId: Joi.string().trim().required(),
-//   result: Joi.object({}).unknown().optional(),
-//   time: Joi.number().default(0).required(),
-//   status: Joi.boolean().default(false).required(),
-// });

package.json

@@ -69,6 +69,7 @@
     "lodash": "^4.17.15",
     "lusca": "~1.6.1",
     "method-override": "~3.0.0",
+    "moment": "^2.24.0",
     "mongoose": "~5.9.3",
     "morgan": "^1.9.1",
     "multer": "~1.4.2",