Merge pull request #14 from KEKDATA/feature/Profile

Feature/profile
KEKDATA · May 29, 2020 · 62bdd3d · 62bdd3d
2 parents 899f923 + f5cff2c
commit 62bdd3d
Show file tree

Hide file tree

Showing 54 changed files with 1,055 additions and 200 deletions.
diff --git a/backend/Procfile b/backend/Procfile
@@ -3,3 +3,4 @@ parser: ts-node --files src/queues/parser.ts
 sentiment: ts-node --files src/queues/sentiment.ts
 bayes: ts-node --files src/queues/bayes.ts
 merge: ts-node --files src/queues/merge.ts
+profile: ts-node --files src/queues/profile.ts
diff --git a/backend/package.json b/backend/package.json
@@ -10,6 +10,7 @@
     "sentiment": "ts-node --files src/queues/sentiment.ts",
     "bayes": "ts-node --files src/queues/bayes.ts",
     "mege": "ts-node --files src/queues/merge.ts",
+    "profile": "ts-node --files src/queues/profile.ts",
     "tsc": "tsc --watch",
     "test": "echo \"Error: no test specified\" && exit 1",
     "fix-code": "prettier --write './src/**/*.ts'",

diff --git a/backend/src/lib/regex/check_is_link/index.ts b/backend/src/lib/regex/check_is_link/index.ts
@@ -0,0 +1,8 @@
+export const checkIsLink = (link: string | undefined) => {
+  if (!link) {
+    return false;
+  }
+
+  const isLink = /(https?:\/\/[^\s]+)/g.test(link);
+  return isLink;
+};
diff --git a/backend/src/queues/bayes.ts b/backend/src/queues/bayes.ts
@@ -10,7 +10,10 @@ const bayesQueue = new Queue('bayes', OPTIONS);
 const callbackQueue = new Queue('callback', OPTIONS);
 
 bayesQueue.process(MAX_JOBS_PER_WORKER, job => {
-  const { normalizedTweetsForAnalysis, id } = job.data;
+  const {
+    normalizedTweetsForAnalysis,
+    id,
+  }: { normalizedTweetsForAnalysis: Array<string>; id: string } = job.data;
 
   const tweetsWithBayesClassifier = getTextWithBayesClassifier(
     normalizedTweetsForAnalysis,

diff --git a/backend/src/queues/merge.ts b/backend/src/queues/merge.ts
@@ -4,6 +4,7 @@ import { nanoid } from 'nanoid';
 import { insertionSentimentTweetsSort } from '../twitter/tweets/lib/insertion_sentiment_tweets_sort';
 
 import { OPTIONS, MAX_JOBS_PER_WORKER } from './config';
+import { Send } from '../types';
 
 const callbackQueue = new Queue('callback', OPTIONS);
 const webQueue = new Queue('web', OPTIONS);
@@ -12,7 +13,7 @@ const webQueue = new Queue('web', OPTIONS);
 const jobsProgress = new Map();
 
 callbackQueue.process(MAX_JOBS_PER_WORKER, job => {
-  const { jobId, options } = job.data;
+  const { jobId, options }: { jobId: string; options: Send } = job.data;
 
   const jobOptions = jobsProgress.get(jobId);
 

diff --git a/backend/src/queues/parser.ts b/backend/src/queues/parser.ts
@@ -5,9 +5,10 @@ import { setupWebdriverFx } from '../webdriver';
 import { getTextWithAlphaOnly } from '../lib/normalizers/alphabet';
 
 import { createdTwitterParse } from '../twitter/twitter_parse';
-import { Tweet } from '../twitter/types';
+import { ParsedTweets, TweetsTabs } from '../twitter/types';
 
 import { OPTIONS, MAX_JOBS_PER_WORKER } from './config';
+import { Send } from '../types';
 
 console.info('Parser connected');
 
@@ -17,12 +18,16 @@ const bayesQueue = new Queue('bayes', OPTIONS);
 const callbackQueue = new Queue('callback', OPTIONS);
 
 parserQueue.process(MAX_JOBS_PER_WORKER, async job => {
-  const { id, options } = job.data;
+  const {
+    id,
+    options,
+    tweetsType,
+  }: { tweetsType: TweetsTabs; id: string; options: Send } = job.data;
 
   await setupWebdriverFx({ options });
   const {
     parsedTweets,
-  }: { parsedTweets: Array<Tweet> } = await createdTwitterParse(null);
+  }: { parsedTweets: ParsedTweets } = await createdTwitterParse(tweetsType);
 
   callbackQueue.add({ jobId: id, options: { parsedTweets } });
 

diff --git a/backend/src/queues/profile.ts b/backend/src/queues/profile.ts
@@ -0,0 +1,49 @@
+import Queue from 'bull';
+import { nanoid } from 'nanoid';
+
+import { OPTIONS, MAX_JOBS_PER_WORKER } from './config';
+import { getProfileInfo } from '../twitter/profile/profile_info';
+import { setupWebdriverFx } from '../webdriver';
+import { NormalizedProfileInfo, ProfileParsedInfo, Send } from '../types';
+import { ProfileInfo } from '../twitter/types';
+
+console.info('Profile connected');
+
+const profileQueue = new Queue('profile', OPTIONS);
+const webQueue = new Queue('web', OPTIONS);
+
+profileQueue.process(MAX_JOBS_PER_WORKER, async job => {
+  const {
+    id,
+    options,
+    tweetsType,
+  }: { tweetsType: ProfileInfo; id: string; options: Send } = job.data;
+
+  await setupWebdriverFx({ options });
+
+  const {
+    profileInfo,
+  }: { profileInfo: ProfileParsedInfo } = await getProfileInfo(null);
+
+  const { activityInfo, contactInfo, ...actualProfile } = profileInfo;
+  const normalizedActivityInfo = activityInfo.map((info: string) => ({
+    id: nanoid(),
+    info,
+  }));
+  const normalizedContactInfo = contactInfo.map((info: string) => ({
+    id: nanoid(),
+    info,
+  }));
+
+  const normalizedProfileInfo: NormalizedProfileInfo = {
+    activityInfo: normalizedActivityInfo,
+    contactInfo: normalizedContactInfo,
+    ...actualProfile,
+    tweetsType,
+  };
+
+  webQueue.add({
+    id,
+    result: normalizedProfileInfo,
+  });
+});
diff --git a/backend/src/queues/sentiment.ts b/backend/src/queues/sentiment.ts
@@ -10,7 +10,10 @@ const sentimentQueue = new Queue('sentiment', OPTIONS);
 const callbackQueue = new Queue('callback', OPTIONS);
 
 sentimentQueue.process(MAX_JOBS_PER_WORKER, job => {
-  const { normalizedTweetsForAnalysis, id } = job.data;
+  const {
+    normalizedTweetsForAnalysis,
+    id,
+  }: { normalizedTweetsForAnalysis: Array<string>; id: string } = job.data;
 
   const {
     dataWithSentiments: tweetsWithSentiments,

diff --git a/backend/src/queues/start_parser_queues.ts b/backend/src/queues/start_parser_queues.ts
@@ -2,18 +2,31 @@ import Queue from 'bull';
 import { nanoid } from 'nanoid';
 
 import { Send } from '../types';
-import { SEARCH_TWEETS_TARGET } from '../twitter/constants/type_parse_target';
+import {
+  SEARCH_TWEETS_TARGET,
+  PROFILE_TARGET,
+} from '../twitter/constants/type_parse_target';
 import { sendFx } from '../socket';
 
 import { OPTIONS, MAX_JOBS_PER_WORKER } from './config';
+import {
+  TWEETS_TAB,
+  TWEETS_REPLIES_TAB,
+  PROFILE_INFO_TYPE,
+} from '../twitter/constants/tabs';
+import {
+  LATEST_TWEETS,
+  TOP_TWEETS,
+} from '../../../frontend/src/constants/tweets_types';
 
 const parserQueue = new Queue(`parser`, OPTIONS);
 const callbackQueue = new Queue('callback', OPTIONS);
-const socketSendQueue = new Queue('web', OPTIONS);
+const webQueue = new Queue('web', OPTIONS);
+const profileQueue = new Queue('profile', OPTIONS);
 
 console.info('start parser queues connected');
 
-socketSendQueue.process(MAX_JOBS_PER_WORKER, job => {
+webQueue.process(MAX_JOBS_PER_WORKER, job => {
   const { id, result } = job.data;
 
   sendFx({
@@ -25,14 +38,62 @@ socketSendQueue.process(MAX_JOBS_PER_WORKER, job => {
 export const startParserQueues = (message: { options: Send; id: string }) => {
   const { options, id } = message;
 
-  const { tweetsSettings, parseUrl, parseTarget } = options;
+  const { tweetsSettings, profileSettings, parseUrl, parseTarget } = options;
+  const processName = `parse:${id}`;
+
+  if (parseTarget === PROFILE_TARGET) {
+    profileQueue.add({
+      id,
+      options,
+      tweetsType: PROFILE_INFO_TYPE,
+    });
+
+    if (profileSettings && profileSettings.isTweets) {
+      const tweetsType = TWEETS_TAB;
+
+      console.log(`${processName}, ${tweetsType}`);
+
+      setTimeout(() => {
+        const jobId = nanoid();
+
+        callbackQueue.add({
+          jobId,
+          options: { tweetsType, id },
+        });
+        parserQueue.add({
+          id: jobId,
+          options,
+          tweetsType,
+        });
+      });
+    }
+
+    if (profileSettings && profileSettings.isTweetsAndReplies) {
+      const tweetsType = TWEETS_REPLIES_TAB;
+
+      console.log(`${processName}, ${tweetsType}`);
+
+      setTimeout(() => {
+        const jobId = nanoid();
+
+        callbackQueue.add({
+          jobId,
+          options: { tweetsType, id },
+        });
+        parserQueue.add({
+          id: jobId,
+          options,
+          tweetsType,
+        });
+      });
+    }
+  }
 
   if (parseTarget === SEARCH_TWEETS_TARGET) {
     if (tweetsSettings && tweetsSettings.isTop) {
-      const processName = `parse:${id}`;
-      const tweetsType = 'top';
+      const tweetsType = TOP_TWEETS;
 
-      console.log(processName);
+      console.log(`${processName}, ${tweetsType}`);
 
       setTimeout(() => {
         const jobId = nanoid();
@@ -44,21 +105,20 @@ export const startParserQueues = (message: { options: Send; id: string }) => {
         parserQueue.add({
           id: jobId,
           options,
-          processName,
+          tweetsType,
         });
       });
     }
 
     if (tweetsSettings && tweetsSettings.isLatest) {
-      const processName = `parse:${id}`;
       const actualParseUrl = `${parseUrl}&f=live`;
-      const tweetsType = 'latest';
+      const tweetsType = LATEST_TWEETS;
       const actualOptions = {
         ...options,
         parseUrl: actualParseUrl,
       };
 
-      console.log(processName);
+      console.log(`${processName}, ${tweetsType}`);
 
       setTimeout(() => {
         const jobId = nanoid();
@@ -68,9 +128,9 @@ export const startParserQueues = (message: { options: Send; id: string }) => {
           options: { tweetsType, id },
         });
         parserQueue.add({
-          processName,
           id: jobId,
           options: actualOptions,
+          tweetsType,
         });
       });
     }

diff --git a/backend/src/socket.ts b/backend/src/socket.ts
@@ -11,7 +11,7 @@ import { nanoid } from 'nanoid';
 
 import { startParserQueues } from './queues/start_parser_queues';
 import { FinalTweet } from './twitter/types';
-import { Send } from './types';
+import { NormalizedProfileInfo, Send } from './types';
 
 // TODO: Чистить со временем или после завершения работы нужных процессов.
 let socketCollection: { [id: string]: WebSocket.Server } = {};
@@ -20,13 +20,15 @@ export const $socketMessage = createStore<Send | {}>({});
 
 export const sendFx: Effect<
   {
-    result: {
-      finalTweets: Array<FinalTweet>;
-      meanSentiment: number;
-      minCoefficient: FinalTweet;
-      maxCoefficient: FinalTweet;
-      tweetsType: 'top' | 'latest';
-    };
+    result:
+      | {
+          finalTweets: Array<FinalTweet>;
+          meanSentiment: number;
+          minCoefficient: FinalTweet;
+          maxCoefficient: FinalTweet;
+          tweetsType: 'top' | 'latest';
+        }
+      | NormalizedProfileInfo;
     id: string;
   },
   any

diff --git a/backend/src/twitter/constants/tabs.ts b/backend/src/twitter/constants/tabs.ts
@@ -1,6 +1,7 @@
-import { Likes, Media, Tweets, TweetsAndReplies } from '../types';
+import { Likes, Media, ProfileInfo, Tweets, TweetsAndReplies } from '../types';
 
-export const TWEETS_TAB: Tweets = 'Tweets';
-export const TWEETS_REPLIES_TAB: TweetsAndReplies = 'TweetsAndReplies';
-export const MEDIA: Media = 'Media';
-export const LIKES: Likes = 'Likes';
+export const TWEETS_TAB: Tweets = 'tweets';
+export const TWEETS_REPLIES_TAB: TweetsAndReplies = 'tweetsAndReplies';
+export const MEDIA: Media = 'media';
+export const LIKES: Likes = 'likes';
+export const PROFILE_INFO_TYPE: ProfileInfo = 'profile_info';
diff --git a/backend/src/twitter/constants/type_parse_target.ts b/backend/src/twitter/constants/type_parse_target.ts
@@ -1,2 +1,4 @@
-export const PROFILE_TARGET: string = 'profile';
-export const SEARCH_TWEETS_TARGET: string = 'search_tweets';
+import { ProfileOption, SearchTweetsOption } from '../types';
+
+export const PROFILE_TARGET: ProfileOption = 'profile';
+export const SEARCH_TWEETS_TARGET: SearchTweetsOption = 'search_tweets';
diff --git a/backend/src/twitter/model.ts b/backend/src/twitter/model.ts
@@ -3,7 +3,13 @@ import { Page, Browser } from 'playwright';
 
 import { ProfileTabs } from './types';
 
-import { LIKES, MEDIA, TWEETS_REPLIES_TAB, TWEETS_TAB } from './constants/tabs';
+import {
+  LIKES,
+  MEDIA,
+  TWEETS_REPLIES_TAB,
+  TWEETS_TAB,
+  PROFILE_INFO_TYPE,
+} from './constants/tabs';
 import { setupWebdriverFx } from '../webdriver';
 
 export const $profileTab = createStore<ProfileTabs | ''>('');
@@ -17,6 +23,7 @@ export const $isProfileTarget = $profileTab.map((tab: ProfileTabs) => {
     TWEETS_REPLIES_TAB,
     MEDIA,
     LIKES,
+    PROFILE_INFO_TYPE,
   ].includes(tab);
 
   return isProfileTarget;

diff --git a/backend/src/twitter/profile/constants/selectors.ts b/backend/src/twitter/profile/constants/selectors.ts
@@ -23,3 +23,6 @@ export const PROFILE_TABS: string = '[role="tablist"].r-1pi2tsx.r-hbs49y';
 export const REPLIES_LINK_SELECTOR: string = `${PROFILE_TABS} > div:nth-child(2) > a`;
 export const MEDIA_LINK_SELECTOR: string = `${PROFILE_TABS} > div:nth-child(3) > a`;
 export const LIKES_LINK_SELECTOR: string = `${PROFILE_TABS} > div:nth-child(4) > a`;
+
+export const PROFILE_IMAGE_CONTAINER_SELECTOR: string =
+  '.css-1dbjc4n.r-obd0qt.r-18u37iz.r-1w6e6rj.r-1wtj0ep';