Merge pull request #120 from fsluis/master

Streaming export API

Gemfile

@@ -5,4 +5,8 @@ platforms :rbx do
   gem 'rubinius-developer_tools'
 end
 
+group :development, :test do
+  gem 'webmock'
+end
+
 gemspec

README.markdown

@@ -190,7 +190,26 @@ of JSON objects rather than a single JSON array.
 
 For example, dumping list members via the "list" method works like this:
 
-    gibbon_export.list({:id => list_id})
+    gibbon_export.list({:id => *list_id*})
+
+One can also use this in a streaming fashion, where each row is parsed on it comes in like this:
+
+    gibbon_export.list({:id => *list_id*}) { |row| *do_sth_with* row }
+
+For the streaming functionality, it is important to supply an explicit block / procedure to the export functions, not an implicit one. So, the preceding and following one will work. Please note this method also includes a counter (*i*, starting at 0) telling which row of data you're receiving:
+```
+    method = Proc.new do |row, i|
+        *do_sth_with* row
+    end
+    gibbon_export.list(params, &method)
+```
+
+Please note, the following example gives a block that is outside of the function and therefore **won't** work:
+```
+    gibbon_export.list({:id => *list_id*}) do |row|
+        *do_sth_with* row
+    end
+```
 
 ##Thanks
 

gibbon.gemspec

@@ -1,9 +1,10 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
+require 'gibbon/version'
 
 Gem::Specification.new do |s|
   s.name        = "gibbon"
-  s.version     = "1.1.5"
+  s.version     = Gibbon::VERSION
   s.authors     = ["Amro Mousa"]
   s.email       = ["amromousa@gmail.com"]
   s.homepage    = "http://github.com/amro/gibbon"

lib/gibbon/api_category.rb

@@ -27,27 +27,26 @@ def call(method, params = {})
       headers = params.delete(:headers) || {}
       response = self.class.post(api_url, :body => MultiJson.dump(params), :headers => headers, :timeout => @timeout)
 
-      parsed_response = nil
-
-      if (response.body)
-        begin
-          parsed_response = MultiJson.load(response.body)
-        rescue MultiJson::ParseError
-          parsed_response = {
-            "error" => "Unparseable response: #{response.body}", 
-            "name" => "UNPARSEABLE_RESPONSE", 
-            "code" => 500
-          }
-        end
-
-        if should_raise_for_response?(parsed_response)
-          error = MailChimpError.new(parsed_response["error"])
-          error.code = parsed_response["code"]
-          error.name = parsed_response["name"]
-          raise error
-        end
+      parse_response(response.body) if response.body
+    end
+
+    def parse_response(response, check_error = true)
+      begin
+        parsed_response = MultiJson.load(response)
+      rescue MultiJson::ParseError
+        parsed_response = {
+          "error" => "Unparseable response: #{response}",
+          "name" => "UNPARSEABLE_RESPONSE",
+          "code" => 500
+        }
       end
 
+      if should_raise_for_response?(parsed_response)
+        error = MailChimpError.new(parsed_response["error"])
+        error.code = parsed_response["code"]
+        error.name = parsed_response["name"]
+        raise error
+      end
       parsed_response
     end
 
@@ -100,7 +99,6 @@ def get_data_center_from_api_key
       data_center
     end
 
-
     private
 
     def ensure_api_key(params)

lib/gibbon/export.rb

@@ -14,28 +14,47 @@ def export_api_url
       "http://#{get_data_center_from_api_key}api.mailchimp.com/export/1.0/"
     end
 
-    def call(method, params = {})
-      ensure_api_key params
+    # fsluis: Alternative, streaming, interface to mailchimp export api
+    #         Prevents having to keep lots of data in memory
+    def call(method, params = {}, &block)
+      rows = []
 
       api_url = export_api_url + method + "/"
       params = @default_params.merge(params).merge({:apikey => @api_key})
-      response = self.class.post(api_url, :body => MultiJson.dump(params), :timeout => @timeout)
-
-      lines = response.body.lines
-      if @throws_exceptions
-        # ignore blank responses
-        return [] if !lines.first || lines.first.strip.empty?
-
-        first_line = MultiJson.load(lines.first) if lines.first
+      block = Proc.new { |row| rows << row } unless block_given?
+      ensure_api_key params
 
-        if should_raise_for_response?(first_line)
-          error = MailChimpError.new(first_line["error"])
-          error.code = first_line["code"]
-          raise error
+      url = URI.parse(api_url)
+      req = Net::HTTP::Post.new(url.path, initheader = {'Content-Type' => 'application/json'})
+      req.body = MultiJson.dump(params)
+      puts params
+      Net::HTTP.start(url.host, url.port, :read_timeout => @timeout) do |http|
+        # http://stackoverflow.com/questions/29598196/ruby-net-http-read-body-nethttpokread-body-called-twice-ioerror
+        http.request req do |response|
+          i = -1
+          last = ''
+          response.read_body do |chunk|
+            #puts "Chunk length: #{chunk.length}"
+            #puts "Chunk: #{chunk}"
+            next if chunk.nil? or chunk.strip.empty?
+            lines = (last+chunk).split("\n")
+            # There seems to be a bug (?) in the export API. Sometimes there's not a newline in between json objects...
+            lines = split_json(lines, '][')
+            lines = split_json(lines, '}{')
+            last = lines.pop || ''
+            lines.each do |line|
+              block.call(parse_response(line, i<0), i+=1) unless line.nil?
+            end
+          end
+          block.call(parse_response(last, i<0), i+=1) unless last.nil? or last.empty?
         end
       end
+      rows unless block_given?
+    end
 
-      lines
+    def parse_response(res, check_error)
+      return [] if res.strip.empty?
+      super(res, check_error)
     end
 
     def set_instance_defaults
@@ -44,7 +63,8 @@ def set_instance_defaults
       super
     end
 
-    def method_missing(method, *args)
+    # fsluis: added a &block to this method and function call
+    def method_missing(method, *args, &block)
       # To support underscores, we camelize the method name
 
       # Thanks for the camelize gsub, Rails
@@ -55,7 +75,7 @@ def method_missing(method, *args)
       # must be upcased (See "Campaign Report Data Methods" in their API docs).
       method = method[0].chr.downcase + method[1..-1].gsub(/aim$/i, 'AIM')
 
-      call(method, *args)
+      call(method, *args, &block)
     end
 
     def respond_to_missing?(method, include_private = false)
@@ -65,6 +85,16 @@ def respond_to_missing?(method, include_private = false)
 
     private
 
+    def split_json(lines, delimiter)
+      lines.map do |line|
+        if line.include? delimiter
+          line.split(delimiter).each_with_index.map{ |s, i| i%2==0 ? s+delimiter[0] : delimiter[1]+s }
+        else
+          line
+        end
+      end.flatten
+    end
+
     def ensure_api_key(params)
       unless @api_key || @default_params[:apikey] || params[:apikey]
         raise Gibbon::GibbonError, "You must set an api_key prior to making a call"

lib/gibbon/version.rb

@@ -0,0 +1,3 @@
+module Gibbon
+  VERSION = "1.1.6"
+end

spec/gibbon/gibbon_spec.rb

@@ -1,5 +1,7 @@
 require 'spec_helper'
 require 'cgi'
+require 'webmock'
+require 'webmock/rspec'
 
 describe Gibbon do
 
@@ -229,19 +231,24 @@
       @gibbon = Gibbon::Export.new(@key)
       @url = "http://us1.api.mailchimp.com/export/1.0/"
       @body = {:apikey => @key, :id => "listid"}
-      @returns = Struct.new(:body).new(MultiJson.dump(["array", "entries"]))
+      @return_items = ["array", "entries"]
+      @returns = MultiJson.dump(@return_items)
     end
 
     it "handle api key with dc" do
       @api_key = "TESTKEY-us2"
       @gibbon = Gibbon::Export.new(@api_key)
 
       @body[:apikey] = @api_key
-      params = {:body => MultiJson.dump(@body), :timeout => 30}
-
       url = @url.gsub('us1', 'us2') + "sayHello/"
-      expect(Gibbon::Export).to receive(:post).with(url, params).and_return(@returns)
+
+      # Fake request
+      stub_request(:post, url).
+        to_return(:body => @returns, :status => 200)
+
+      # Check request url
       @gibbon.say_hello(@body)
+      expect(WebMock).to have_requested(:post, url).with(:body => @body)
     end
 
     it "uses timeout if set" do
@@ -251,34 +258,52 @@
 
     it "not throw exception if the Export API replies with a JSON hash containing a key called 'error'" do
       @gibbon.throws_exceptions = false
-      allow(Gibbon::Export).to receive(:post).and_return(Struct.new(:body).new(MultiJson.dump({'error' => 'bad things'})))
+      reply = MultiJson.dump({:error => 'bad things'})
+      stub_request(:post, @url + 'sayHello/').
+        to_return(:body => reply, :status => 200)
 
       @gibbon.say_hello(@body)
     end
 
     it "throw exception if configured to and the Export API replies with a JSON hash containing a key called 'error'" do
       @gibbon.throws_exceptions = true
-      params = {:body => @body, :timeout => 30}
-      reply = Struct.new(:body).new MultiJson.dump({'error' => 'bad things', 'code' => '123'})
-      allow(Gibbon::Export).to receive(:post).and_return reply
+      reply = MultiJson.dump({:error => 'bad things', :code => '123'})
+      stub_request(:post, @url + 'sayHello/').
+        to_return(:body => reply, :status => 200)
 
       expect {@gibbon.say_hello(@body)}.to raise_error(Gibbon::MailChimpError)
     end
 
     it "should handle a single empty space response without throwing an exception" do
       @gibbon.throws_exceptions = true
-      allow(Gibbon::Export).to receive(:post).and_return(Struct.new(:body).new(" "))
+      stub_request(:post, @url + 'sayHello/').
+        to_return(:body => " ", :status => 200)
+      #allow(Gibbon::Export).to receive(:post).and_return(Struct.new(:body).new(" "))
 
       expect(@gibbon.say_hello(@body)).to eq([])
     end
 
     it "should handle an empty response without throwing an exception" do
       @gibbon.throws_exceptions = true
-      allow(Gibbon::Export).to receive(:post).and_return(Struct.new(:body).new(""))
+      stub_request(:post, @url + 'sayHello/').
+        to_return(:body => "", :status => 200)
+      #allow(Gibbon::Export).to receive(:post).and_return(Struct.new(:body).new(""))
 
       expect(@gibbon.say_hello(@body)).to eq([])
     end
 
+    it "should feed API results per row to a given block" do
+      # Fake request
+      stub_request(:post, @url + 'sayHello/').
+        to_return(:body => @returns, :status => 200)
+
+      # Check request url
+      @result = []
+      @gibbon.say_hello(@body) { |res| @result << res }
+      expect(@result).to contain_exactly(@return_items)
+    end
+
+
   end
 
   private