Added reply by tweet feature

Gemfile

@@ -70,6 +70,8 @@ gem 'mailman', require: false
 # To convert html to markdown
 gem 'reverse_markdown'
 
+gem 'twitter'
+
 # run with `bundle install --without production` or `bundle install --without mysql` to exclude this
 group :mysql, :production do
   gem 'mysql2', '>= 0.4.4'

Gemfile.lock

@@ -65,6 +65,7 @@ GEM
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
     bindex (0.5.0)
+    buftok (0.2.0)
     builder (3.2.3)
     byebug (11.0.0)
     chronic (0.10.2)
@@ -147,10 +148,19 @@ GEM
     grape-swagger-ui (2.2.8)
       railties (>= 3.1)
     hashie (3.6.0)
-    highline (2.0.1)
+    highline (2.0.0)
+    hoe (3.17.0)
+      rake (>= 0.8, < 13.0)
+    http (3.3.0)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.0)
+      http_parser.rb (~> 0.6.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
+    http-form_data (2.1.1)
     http_accept_language (2.1.1)
+    http_parser.rb (0.6.0)
     i18n (1.5.3)
       concurrent-ruby (~> 1.0)
     i18n-js (3.2.1)
@@ -207,6 +217,8 @@ GEM
       maildir (>= 0.5.0)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
+    memoizable (0.4.2)
+      thread_safe (~> 0.3, >= 0.3.1)
     metaclass (0.0.4)
     method_source (0.9.2)
     mime-types (3.2.2)
@@ -230,6 +242,7 @@ GEM
     mustermann-grape (1.0.0)
       mustermann (~> 1.0.0)
     mysql2 (0.5.2)
+    naught (1.1.0)
     netrc (0.11.0)
     nifty-generators (0.4.6)
     nio4r (2.3.1)
@@ -407,6 +420,7 @@ GEM
       rack (>= 1.5.0)
       rack-protection (>= 1.5.0)
       redis (>= 3.3.5, < 5)
+    simple_oauth (0.3.1)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -445,6 +459,17 @@ GEM
     turbolinks (5.2.0)
       turbolinks-source (~> 5.2)
     turbolinks-source (5.2.0)
+    twitter (6.2.0)
+      addressable (~> 2.3)
+      buftok (~> 0.2.0)
+      equalizer (~> 0.0.11)
+      http (~> 3.0)
+      http-form_data (~> 2.0)
+      http_parser.rb (~> 0.6.0)
+      memoizable (~> 0.4.0)
+      multipart-post (~> 2.0)
+      naught (~> 1.0)
+      simple_oauth (~> 0.3.0)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     uglifier (4.1.20)
@@ -560,6 +585,7 @@ DEPENDENCIES
   therubyracer
   timecop
   turbolinks (~> 5)
+  twitter
   tzinfo-data
   uglifier (>= 1.0.3)
   unicode-emoji

app/models/comment.rb

@@ -344,6 +344,102 @@ def trimmed_content?
     comment.include?(COMMENT_FILTER)
   end
 
+  def self.receive_tweet
+    comments = Comment.where.not(tweet_id: nil)
+    if comments.any?
+      receive_tweet_using_since comments
+    else
+      receive_tweet_without_using_since
+    end
+  end
+
+  def self.receive_tweet_using_since(comments)
+    comment = comments.last
+    since_id = comment.tweet_id
+    tweets = Client.search(ENV["TWEET_SEARCH"], since_id: since_id).collect do |tweet|
+      tweet
+    end
+    tweets.each do |tweet|
+      puts tweet.text
+    end
+    tweets = tweets.reverse
+    check_and_add_tweets tweets
+  end
+
+  def self.receive_tweet_without_using_since
+    tweets = Client.search(ENV["TWEET_SEARCH"]).collect do |tweet|
+      tweet
+    end
+    tweets = tweets.reverse
+    check_and_add_tweets tweets
+    tweets.each do |tweet|
+      puts tweet.text
+    end
+  end
+
+  def self.check_and_add_tweets(tweets)
+    tweets.each do |tweet|
+      if tweet.reply?
+        in_reply_to_tweet_id = tweet.in_reply_to_tweet_id
+        if in_reply_to_tweet_id.class == Fixnum
+          parent_tweet = Client.status(in_reply_to_tweet_id, tweet_mode: "extended")
+          parent_tweet_full_text = parent_tweet.attrs[:text] || parent_tweet.attrs[:full_text]
+          urls = URI.extract(parent_tweet_full_text)
+          node = get_node_from_urls_present(urls)
+          unless node.nil?
+            twitter_user_name = tweet.user.screen_name
+            tweet_email = find_email(twitter_user_name)
+            users = User.where(email: tweet_email)
+            if users.any?
+              user = users.first
+              replied_tweet_text = tweet.text
+              if tweet.truncated?
+                replied_tweet = Client.status(tweet.id, tweet_mode: "extended")
+                replied_tweet_text = replied_tweet.attrs[:text] || replied_tweet.attrs[:full_text]
+              end
+              replied_tweet_text = replied_tweet_text.gsub(/@(\S+)/){|m| "[#{m}](https://twitter.com/#{m})"}
+              replied_tweet_text = replied_tweet_text.gsub('@','')
+              comment = node.add_comment(uid: user.uid, body: replied_tweet_text, comment_via: 2, tweet_id: tweet.id)
+              comment.notify user
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def self.get_node_from_urls_present(urls)
+    urls.each do |url|
+      if url.include? "https://"
+        if url.last == "."
+          url = url[0...url.length-1]
+        end
+        response = Net::HTTP.get_response(URI(url))
+        redirected_url = response['location']
+        if redirected_url != nil && redirected_url.include?(ENV["WEBSITE_HOST_PATTERN"])
+          node_id = redirected_url.split("/")[-1]
+          if !node_id.nil?
+            node = Node.where(nid: node_id.to_i)
+            if node.any?
+              return node.first
+            end
+          end
+        end
+      end
+
+    end
+    return nil
+  end
+
+  def self.find_email(twitter_user_name)
+    UserTag.all.each do |user_tag|
+      data = user_tag["data"]
+      if data != nil && data["info"] != nil && data["info"]["nickname"] != nil && data["info"]["nickname"].to_s == twitter_user_name
+        return data["info"]["email"]
+      end
+    end
+  end
+
   def parse_quoted_text
     if regex_match = body.match(/(.+)(On .+<.+@.+> wrote:)(.+)/m)
       {

app/models/node.rb

@@ -590,7 +590,8 @@ def add_comment(params = {})
                     thread: thread,
                     timestamp: DateTime.now.to_i,
                     comment_via: comment_via_status,
-                    message_id: params[:message_id])
+                    message_id: params[:message_id],
+                    tweet_id: params[:tweet_id])
     c.save
     c
   end

config/initializers/twitter_client.rb

@@ -0,0 +1,7 @@
+require 'twitter'
+::Client = Twitter::REST::Client.new do |config|
+  config.consumer_key        = ENV["TWITTER_CONSUMER_KEY"]
+  config.consumer_secret     = ENV["TWITTER_CONSUMER_SECRET"]
+  config.access_token        = ENV["TWITTER_ACCESS_TOKEN"]
+  config.access_token_secret = ENV["TWITTER_ACCESS_TOKEN_SECRET"]
+end

config/schedule.rb

@@ -26,6 +26,12 @@
 env :REDIS_URL, ENV['REDIS_URL']
 
 # Cron Job log file
+
+set :bundle_command, 'bundle exec'
+job_type :runner,  "cd :path && :bundle_command rails runner -e :environment ':task' :output"
+
+ENV.each { |k, v| env(k, v) }
+
 set :output, "#{Dir.pwd}/public/cron_log.log"
 
 # To simply print date into the log file for checking if cron job is working properly
@@ -34,6 +40,11 @@
   command "date -u" #This will print utc time every 1 min in log/cron_log.log file
 end
 
+every 1.minutes do
+	runner "Comment.receive_tweet"
+end
+
+
 every 1.day do
   runner "DigestMailJob.perform_async(0)"
 end

containers/docker-compose-production.yml

@@ -37,6 +37,12 @@ services:
       - OAUTH_FACEBOOK_APP_KEY=${OAUTH_FACEBOOK_APP_KEY}
       - OAUTH_FACEBOOK_APP_SECRET=${OAUTH_FACEBOOK_APP_SECRET}
       - REDIS_URL=redis://redis:6379/0
+      - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY}
+      - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET}
+      - TWITTER_ACCESS_TOKEN=${TWITTER_ACCESS_TOKEN}
+      - TWITTER_ACCESS_TOKEN_SECRET=${TWITTER_ACCESS_TOKEN_SECRET}
+      - WEBSITE_HOST_PATTERN=${WEBSITE_HOST_PATTERN}
+      - TWEET_SEARCH=${TWEET_SEARCH}
     volumes:
       - ..:/app
       - /etc/passwd:/etc/passwd:ro

containers/docker-compose-stable.yml

@@ -37,6 +37,12 @@ services:
       - OAUTH_FACEBOOK_APP_KEY=${OAUTH_FACEBOOK_APP_KEY}
       - OAUTH_FACEBOOK_APP_SECRET=${OAUTH_FACEBOOK_APP_SECRET}
       - REDIS_URL=redis://redis:6379/0
+      - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY}
+      - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET}
+      - TWITTER_ACCESS_TOKEN=${TWITTER_ACCESS_TOKEN}
+      - TWITTER_ACCESS_TOKEN_SECRET=${TWITTER_ACCESS_TOKEN_SECRET}
+      - WEBSITE_HOST_PATTERN=${WEBSITE_HOST_PATTERN}
+      - TWEET_SEARCH=${TWEET_SEARCH}
     volumes:
       - ..:/app
       - /etc/passwd:/etc/passwd:ro

containers/docker-compose-unstable.yml

@@ -37,6 +37,12 @@ services:
       - OAUTH_FACEBOOK_APP_KEY=${OAUTH_FACEBOOK_APP_KEY}
       - OAUTH_FACEBOOK_APP_SECRET=${OAUTH_FACEBOOK_APP_SECRET}
       - REDIS_URL=redis://redis:6379/0
+      - TWITTER_CONSUMER_KEY=${TWITTER_CONSUMER_KEY}
+      - TWITTER_CONSUMER_SECRET=${TWITTER_CONSUMER_SECRET}
+      - TWITTER_ACCESS_TOKEN=${TWITTER_ACCESS_TOKEN}
+      - TWITTER_ACCESS_TOKEN_SECRET=${TWITTER_ACCESS_TOKEN_SECRET}
+      - WEBSITE_HOST_PATTERN=${WEBSITE_HOST_PATTERN}
+      - TWEET_SEARCH=${TWEET_SEARCH}
     volumes:
       - ..:/app
       - /etc/passwd:/etc/passwd:ro

db/migrate/20180809125000_add_tweet_column_to_comment.rb

@@ -0,0 +1,5 @@
+class AddTweetColumnToComment < ActiveRecord::Migration[5.2]
+  def change
+  	add_column :comments, :tweet_id, :string
+  end
+end

db/schema.rb.example

@@ -51,6 +51,8 @@ ActiveRecord::Schema.define(version: 2019_03_01_075323) do
     t.integer "aid",                          default: 0,  null: false
     t.integer "comment_via", limit: 4,          default: 0
     t.string  "message_id",  limit: 255
+    t.string  "tweet_id"
+
   end
 
   add_index "comments", ["comment"], name: "index_comments_on_comment", type: :fulltext if ActiveRecord::Base.connection.adapter_name == 'Mysql2'

doc/reply_by_tweet.md

@@ -0,0 +1,29 @@
+## Reply By Tweet
+
+-  We are using [twitter gem](https://github.com/sferik/twitter "twitter gem") which implements twitter apis and gives functions to easily implement twitter apis.
+
+### Steps:
+
+- Cron job for polling for getting new tweets to `publiclab` is defined in [config/schedule.rb](https://github.com/publiclab/plots2/blob/master/config/schedule.rb "config/schedule.rb") using whenever gem which will call `receive_tweet` function of [models/comment.rb](https://github.com/publiclab/plots2/blob/master/app/models/comment.rb "models/comment.rb") in the interval of one minute.
+
+- `receive_tweet` method of [models/comment.rb](https://github.com/publiclab/plots2/blob/master/app/models/comment.rb "models/comment.rb") will look for if there is any comment already present that contains tweet_id if it does it will call `receive_tweet_using_since` otherwise it will call `receive_tweet_without_using_since `.
+
+- `receive_tweet_using_since` will search for the tweets to the `publiclab` which are tweeted after that tweet with the `tweet_id` present in the database.
+
+- After that check if that tweet is the reply of some other tweet. If it does then find the parent tweet otherwise ignore this tweet and check next tweets untill tweets end.
+
+- After finding parent tweet, search for the links present in the tweets in the form of `https://publiclab.org/n/_____` and find the the node_id present in `https://publiclab.org/n/node_id` if there is any node present in the database with this node_id then search for the user's email who did the tweet using the username and email data present in the hash form in the `data` column otherwise ignore this tweet.
+
+- If that twitter username is present in the `user_tags` column  then add the tweet otherwise ignore the current tweet.
+
+- Same process is with when there are no comment present with tweet_id in the comment table where we search for all the replied tweets to the `publiclab`.
+
+To use this feature we have to set some environment variables which includes Twitter Api keys and Searching Query: For getting Twitter Keys go to [Twitter app keys](https://apps.twitter.com/)
+Environment varible used for twitter keys are : `TWITTER_CONSUMER_KEY`, `TWITTER_CONSUMER_SECRET`, `TWITTER_ACCESS_TOKEN` and `TWITTER_ACCESS_TOKEN_SECRET`.
+
+Other Environment variable used are : `WEBSITE_HOST_PATTERN` which can be like `//publiclab.org/n/` and `TWEET_SEARCH` which is used to search for the tweets and can be like `to:publiclab`.
+
+### Summery
+
+Once a minute, this system scans tweets "to:PublicLab" (or specified search, see below) for URLs matching our shortlink pattern, i.e. "https://publiclab.org/n/_____", where "_____" is a node nid. Each of the returned tweets is added as a comment to the node it responded to.
+