diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 969ef2b..eeaf9fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,7 @@ jobs: - uses: actions/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} + - run: gem install bundler - run: bundle - run: sleep 10 - run: bundle exec rspec diff --git a/.rubocop.yml b/.rubocop.yml index 954b518..30cea9e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,6 +1,7 @@ AllCops: NewCops: enable - TargetRubyVersion: 2.4 + TargetRubyVersion: 2.5 + SuggestExtensions: false Layout/EmptyLineBetweenDefs: EmptyLineBetweenClassDefs: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 95237c9..56f2a7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # CHANGELOG +## v3.3.0 + +* Update httprb +* Changed oj default options +* Allow to set oj json options + ## v3.2.1 * Fix `refresh` having a empty body breaking in elasticsearch 7.11 diff --git a/README.md b/README.md index 279637d..4dcb4b5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ **Full-Featured Elasticsearch Ruby Client with a Chainable DSL** -[![Build Status](https://secure.travis-ci.org/mrkamel/search_flip.svg?branch=master)](http://travis-ci.org/mrkamel/search_flip) +[![Build](https://github.com/mrkamel/search_flip/workflows/test/badge.svg)](https://github.com/mrkamel/search_flip/actions?query=workflow%3Atest+branch%3Amaster) [![Gem Version](https://badge.fury.io/rb/search_flip.svg)](http://badge.fury.io/rb/search_flip) Using SearchFlip it is dead-simple to create index classes that correspond to @@ -51,8 +51,7 @@ CommentIndex.search("hello world").where(available: true).sort(id: "desc").aggre ``` -Finally, SearchFlip comes with a minimal set of dependencies (http-rb, hashie -and oj only). +Finally, SearchFlip comes with a minimal set of dependencies. ## Reference Docs @@ -883,52 +882,46 @@ Thus, if your ORM supports `.find_each`, `#id` and `#where` you are already good to go. Otherwise, simply add your custom implementation of those methods that work with whatever ORM you use. -## Date and Timestamps in JSON +## JSON -Elasticsearch requires dates and timestamps to have one of the formats listed -here: [https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#strict-date-time). - -However, `JSON.generate` in ruby by default outputs something like: +SearchFlip is using the [Oj gem](https://github.com/ohler55/oj) to generate +JSON. More concretely, SearchFlip is using: ```ruby -JSON.generate(time: Time.now.utc) -# => "{\"time\":\"2018-02-22 18:19:33 UTC\"}" +Oj.dump({ key: "value" }, mode: :custom, use_to_json: true, time_format: :xmlschema, bigdecimal_as_decimal: false) ``` -This format is not compatible with Elasticsearch by default. If you're on -Rails, ActiveSupport adds its own `#to_json` methods to `Time`, `Date`, etc. -However, ActiveSupport checks whether they are used in combination with -`JSON.generate` or not and adapt: +The `use_to_json` option is used for maximum compatibility, most importantly +when using rails `ActiveSupport::TimeWithZone` timestamps, which `oj` can not +serialize natively. However, `use_to_json` adds performance overhead. You can +change the json options via: ```ruby -Time.now.utc.to_json -=> "\"2018-02-22T18:18:22.088Z\"" - -JSON.generate(time: Time.now.utc) -=> "{\"time\":\"2018-02-22 18:18:59 UTC\"}" +SearchFlip::Config[:json_options] = { + mode: :custom, + use_to_json: false, + time_format: :xmlschema, + bigdecimal_as_decimal: false +} ``` -SearchFlip is using the [Oj gem](https://github.com/ohler55/oj) to generate -JSON. More concretely, SearchFlip is using: +However, you then have to convert timestamps manually for indexation via e.g.: ```ruby -Oj.dump({ key: "value" }, mode: :custom, use_to_json: true) -``` +class MyIndex + # ... -This mitigates the issues if you're on Rails: + def self.serialize(model) + { + # ... -```ruby -Oj.dump(Time.now, mode: :custom, use_to_json: true) -# => "\"2018-02-22T18:21:21.064Z\"" + created_at: model.created_at.to_time + } + end +end ``` -However, if you're not on Rails, you need to add `#to_json` methods to `Time`, -`Date` and `DateTime` to get proper serialization. You can either add them on -your own, via other libraries or by simply using: - -```ruby -require "search_flip/to_json" -``` +Please check out the oj docs for more details. ## Feature Support diff --git a/lib/search_flip.rb b/lib/search_flip.rb index 6f2c9da..f1ad4df 100644 --- a/lib/search_flip.rb +++ b/lib/search_flip.rb @@ -3,6 +3,7 @@ require "http" require "hashie" require "thread" +require "json" require "oj" require "set" diff --git a/lib/search_flip/bulk.rb b/lib/search_flip/bulk.rb index f6eff9f..11c6cc8 100644 --- a/lib/search_flip/bulk.rb +++ b/lib/search_flip/bulk.rb @@ -143,7 +143,7 @@ def upload return if options[:raise] == false - parsed_response = response.parse + parsed_response = SearchFlip::JSON.parse(response.to_s) return unless parsed_response["errors"] diff --git a/lib/search_flip/config.rb b/lib/search_flip/config.rb index f56fef2..fcadcdb 100644 --- a/lib/search_flip/config.rb +++ b/lib/search_flip/config.rb @@ -5,6 +5,12 @@ module SearchFlip bulk_limit: 1_000, bulk_max_mb: 100, auto_refresh: false, - instrumenter: NullInstrumenter.new + instrumenter: NullInstrumenter.new, + json_options: { + mode: :custom, + use_to_json: true, + time_format: :xmlschema, + bigdecimal_as_decimal: false + } } end diff --git a/lib/search_flip/connection.rb b/lib/search_flip/connection.rb index 5983b9a..9effae0 100644 --- a/lib/search_flip/connection.rb +++ b/lib/search_flip/connection.rb @@ -28,7 +28,11 @@ def initialize(options = {}) def version @version_mutex.synchronize do - @version ||= http_client.headers(accept: "application/json").get("#{base_url}/").parse["version"]["number"] + @version ||= begin + response = http_client.headers(accept: "application/json").get("#{base_url}/") + + SearchFlip::JSON.parse(response.to_s)["version"]["number"] + end end end @@ -40,7 +44,9 @@ def version # @return [Hash] The raw response def cluster_health - http_client.headers(accept: "application/json").get("#{base_url}/_cluster/health").parse + response = http_client.headers(accept: "application/json").get("#{base_url}/_cluster/health") + + SearchFlip::JSON.parse(response.to_s) end # Uses the Elasticsearch Multi Search API to execute multiple search requests @@ -71,7 +77,7 @@ def msearch(criterias) .headers(accept: "application/json", content_type: "application/x-ndjson") .post("#{base_url}/_msearch", body: payload) - raw_response.parse["responses"].map.with_index do |response, index| + SearchFlip::JSON.parse(raw_response.to_s)["responses"].map.with_index do |response, index| SearchFlip::Response.new(criterias[index], response) end end @@ -90,10 +96,11 @@ def msearch(criterias) # @return [Hash] The raw response def update_aliases(payload) - http_client + response = http_client .headers(accept: "application/json", content_type: "application/json") .post("#{base_url}/_aliases", body: SearchFlip::JSON.generate(payload)) - .parse + + SearchFlip::JSON.parse(response.to_s) end # Sends an analyze request to Elasticsearch. Raises @@ -105,10 +112,11 @@ def update_aliases(payload) # @return [Hash] The raw response def analyze(request, params = {}) - http_client + response = http_client .headers(accept: "application/json") .post("#{base_url}/_analyze", json: request, params: params) - .parse + + SearchFlip::JSON.parse(response.to_s) end # Fetches information about the specified index aliases. Raises @@ -124,10 +132,11 @@ def analyze(request, params = {}) # @return [Hash] The raw response def get_aliases(index_name: "*", alias_name: "*") - http_client + response = http_client .headers(accept: "application/json", content_type: "application/json") .get("#{base_url}/#{index_name}/_alias/#{alias_name}") - .parse + + SearchFlip::JSON.parse(response.to_s) end # Returns whether or not the associated Elasticsearch alias already @@ -159,10 +168,11 @@ def alias_exists?(alias_name) # @return [Array] The raw response def get_indices(name = "*", params: {}) - http_client + response = http_client .headers(accept: "application/json", content_type: "application/json") .get("#{base_url}/_cat/indices/#{name}", params: params) - .parse + + SearchFlip::JSON.parse(response.to_s) end alias_method :cat_indices, :get_indices @@ -259,10 +269,11 @@ def update_index_settings(index_name, index_settings) # @return [Hash] The index settings def get_index_settings(index_name) - http_client + response = http_client .headers(accept: "application/json") .get("#{index_url(index_name)}/_settings") - .parse + + SearchFlip::JSON.parse(response.to_s) end # Sends a refresh request to Elasticsearch. Raises @@ -310,7 +321,9 @@ def get_mapping(index_name, type_name: nil) url = type_name ? type_url(index_name, type_name) : index_url(index_name) params = type_name && version.to_f >= 6.7 ? { include_type_name: true } : {} - http_client.headers(accept: "application/json").get("#{url}/_mapping", params: params).parse + response = http_client.headers(accept: "application/json").get("#{url}/_mapping", params: params) + + SearchFlip::JSON.parse(response.to_s) end # Deletes the specified index from Elasticsearch. Raises diff --git a/lib/search_flip/criteria.rb b/lib/search_flip/criteria.rb index cb7a55a..acf98bd 100644 --- a/lib/search_flip/criteria.rb +++ b/lib/search_flip/criteria.rb @@ -608,7 +608,7 @@ def execute! http_request.post("#{target.type_url}/_search", params: request_params, json: request) end - SearchFlip::Response.new(self, http_response.parse) + SearchFlip::Response.new(self, SearchFlip::JSON.parse(http_response.to_s)) rescue SearchFlip::ConnectionError, SearchFlip::ResponseError => e raise e unless failsafe_value diff --git a/lib/search_flip/index.rb b/lib/search_flip/index.rb index de7c084..baf5c01 100644 --- a/lib/search_flip/index.rb +++ b/lib/search_flip/index.rb @@ -455,7 +455,9 @@ def include_type_name? # @return [Hash] The specified document def get(id, params = {}) - connection.http_client.headers(accept: "application/json").get("#{type_url}/#{id}", params: params).parse + response = connection.http_client.headers(accept: "application/json").get("#{type_url}/#{id}", params: params) + + SearchFlip::JSON.parse(response.to_s) end # Retrieves the documents specified by ids from elasticsearch. @@ -471,7 +473,9 @@ def get(id, params = {}) # @return [Hash] The raw response def mget(request, params = {}) - connection.http_client.headers(accept: "application/json").post("#{type_url}/_mget", json: request, params: params).parse + response = connection.http_client.headers(accept: "application/json").post("#{type_url}/_mget", json: request, params: params) + + SearchFlip::JSON.parse(response.to_s) end # Sends an analyze request to Elasticsearch. Raises @@ -483,7 +487,9 @@ def mget(request, params = {}) # @return [Hash] The raw response def analyze(request, params = {}) - connection.http_client.headers(accept: "application/json").post("#{index_url}/_analyze", json: request, params: params).parse + response = connection.http_client.headers(accept: "application/json").post("#{index_url}/_analyze", json: request, params: params) + + SearchFlip::JSON.parse(response.to_s) end # Sends a index refresh request to Elasticsearch. Raises diff --git a/lib/search_flip/json.rb b/lib/search_flip/json.rb index 8fc077d..c168f1d 100644 --- a/lib/search_flip/json.rb +++ b/lib/search_flip/json.rb @@ -1,16 +1,11 @@ module SearchFlip class JSON - @default_options = { - mode: :custom, - use_to_json: true - } - - def self.default_options - @default_options + def self.generate(obj) + Oj.dump(obj, SearchFlip::Config[:json_options]) end - def self.generate(obj) - Oj.dump(obj, default_options) + def self.parse(json) + ::JSON.parse(json) end end end diff --git a/lib/search_flip/to_json.rb b/lib/search_flip/to_json.rb index e79e135..0eeb558 100644 --- a/lib/search_flip/to_json.rb +++ b/lib/search_flip/to_json.rb @@ -1,29 +1 @@ -require "time" -require "date" -require "json" - -class Time - def to_json(*args) - iso8601(6).to_json - end -end - -class Date - def to_json(*args) - iso8601.to_json - end -end - -class DateTime - def to_json(*args) - iso8601(6).to_json - end -end - -if defined?(ActiveSupport) - class ActiveSupport::TimeWithZone - def to_json(*args) - iso8601(6).to_json - end - end -end +warn "[DEPRECATION] Using search_flip/to_json is not neccessary anymore" diff --git a/lib/search_flip/version.rb b/lib/search_flip/version.rb index ef36b76..8c09e6d 100644 --- a/lib/search_flip/version.rb +++ b/lib/search_flip/version.rb @@ -1,3 +1,3 @@ module SearchFlip - VERSION = "3.2.1" + VERSION = "3.3.0" end diff --git a/search_flip.gemspec b/search_flip.gemspec index 5991a19..fc0d469 100644 --- a/search_flip.gemspec +++ b/search_flip.gemspec @@ -37,6 +37,7 @@ Gem::Specification.new do |spec| spec.add_dependency "hashie" spec.add_dependency "http" + spec.add_dependency "json" spec.add_dependency "oj" spec.add_dependency "ruby2_keywords" end diff --git a/spec/search_flip/json_spec.rb b/spec/search_flip/json_spec.rb new file mode 100644 index 0000000..2c0e7ae --- /dev/null +++ b/spec/search_flip/json_spec.rb @@ -0,0 +1,45 @@ +require File.expand_path("../spec_helper", __dir__) + +RSpec.describe SearchFlip::JSON do + describe ".generate" do + it "encodes timestamps correctly" do + Timecop.freeze "2020-06-01 12:00:00 UTC" do + expect(described_class.generate(timestamp: Time.now.utc)).to eq('{"timestamp":"2020-06-01T12:00:00.000Z"}') + end + end + + it "encodes bigdecimals as string" do + expect(described_class.generate(value: BigDecimal(1))).to eq('{"value":"1.0"}') + end + + it "delegates to Oj" do + allow(Oj).to receive(:dump) + + payload = { key: "value" } + + described_class.generate(payload) + + expect(Oj).to have_received(:dump).with(payload, mode: :custom, use_to_json: true, time_format: :xmlschema, bigdecimal_as_decimal: false) + end + + it "generates json" do + expect(described_class.generate(key: "value")).to eq('{"key":"value"}') + end + end + + describe ".parse" do + it "returns the parsed json payload" do + expect(described_class.parse('{"key":"value"}')).to eq("key" => "value") + end + + it "delegates to JSON" do + allow(JSON).to receive(:parse) + + payload = '{"key":"value"}' + + described_class.parse(payload) + + expect(JSON).to have_received(:parse).with(payload) + end + end +end diff --git a/spec/search_flip/to_json_spec.rb b/spec/search_flip/to_json_spec.rb deleted file mode 100644 index 86dec74..0000000 --- a/spec/search_flip/to_json_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -require File.expand_path("../spec_helper", __dir__) -require "search_flip/to_json" - -RSpec.describe "to_json" do - it "uses the correct format for Time" do - Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do - expect(Time.now.utc.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"") - end - end - - it "uses the correct format for Date" do - Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do - expect(Date.today.to_json).to eq("\"2018-01-01\"") - end - end - - it "uses the correct format for DateTime" do - Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do - expect(Time.now.utc.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"") - end - end - - it "uses the correct format for TimeWithZone" do - Timecop.freeze Time.parse("2018-01-01 12:00:00 UTC") do - expect(Time.find_zone("UTC").now.to_json).to eq("\"2018-01-01T12:00:00.000000Z\"") - end - end -end