From 25f386a1927c89a80cc48f132b14a70c00eea698 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 26 Jul 2023 12:42:31 -0700 Subject: [PATCH 1/2] API Generator Signed-off-by: Theo Truong --- .rubocop.yml | 2 + CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 5 + api_generator/.rubocop.yml | 21 +++ api_generator/.ruby-version | 1 + api_generator/USER_GUIDE.md | 48 +++++++ api_generator/gemfile | 17 +++ api_generator/lib/action.rb | 52 +++++++ api_generator/lib/action_generator.rb | 127 ++++++++++++++++++ api_generator/lib/api_generator.rb | 68 ++++++++++ api_generator/lib/base_generator.rb | 46 +++++++ api_generator/lib/index_generator.rb | 38 ++++++ api_generator/lib/namespace_generator.rb | 40 ++++++ api_generator/lib/operation.rb | 49 +++++++ api_generator/lib/parameter.rb | 55 ++++++++ api_generator/lib/spec_generator.rb | 86 ++++++++++++ api_generator/lib/version.rb | 32 +++++ .../templates/action.module.mustache | 50 +++++++ api_generator/templates/action.mustache | 17 +++ api_generator/templates/index.mustache | 67 +++++++++ .../templates/legacy_license_header.txt | 25 ++++ api_generator/templates/license_header.txt | 5 + api_generator/templates/namespace.mustache | 24 ++++ api_generator/templates/spec.mustache | 42 ++++++ lib/opensearch/api/namespace/common.rb | 21 +++ 25 files changed, 939 insertions(+) create mode 100644 api_generator/.rubocop.yml create mode 100644 api_generator/.ruby-version create mode 100644 api_generator/USER_GUIDE.md create mode 100644 api_generator/gemfile create mode 100644 api_generator/lib/action.rb create mode 100644 api_generator/lib/action_generator.rb create mode 100644 api_generator/lib/api_generator.rb create mode 100644 api_generator/lib/base_generator.rb create mode 100644 api_generator/lib/index_generator.rb create mode 100644 api_generator/lib/namespace_generator.rb create mode 100644 api_generator/lib/operation.rb create mode 100644 api_generator/lib/parameter.rb create mode 100644 api_generator/lib/spec_generator.rb create mode 100644 api_generator/lib/version.rb create mode 100644 api_generator/templates/action.module.mustache create mode 100644 api_generator/templates/action.mustache create mode 100644 api_generator/templates/index.mustache create mode 100644 api_generator/templates/legacy_license_header.txt create mode 100644 api_generator/templates/license_header.txt create mode 100644 api_generator/templates/namespace.mustache create mode 100644 api_generator/templates/spec.mustache diff --git a/.rubocop.yml b/.rubocop.yml index 90ed82cf3..84c8c865a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -7,6 +7,8 @@ require: AllCops: TargetRubyVersion: 2.5 NewCops: enable + Exclude: + - 'api_generator/**/*' RSpec/ImplicitExpect: Enabled: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 2810e9dcb..5027cb39c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] ### Added - Added `remote_store.restore` action ([#176](https://github.com/opensearch-project/opensearch-ruby/pull/176)) +- Added API Generator ([#139](https://github.com/opensearch-project/opensearch-ruby/issues/139)) ### Changed - Merged `opensearch-transport`, `opensearch-api`, and `opensearch-dsl` into `opensearch-ruby` ([#133](https://github.com/opensearch-project/opensearch-ruby/issues/133)) - Bumped `mocha` gem from 1.x.x to 2.x.x ([#178](https://github.com/opensearch-project/opensearch-ruby/pull/178)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 7c7128eb8..d1f4abcc9 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -6,6 +6,7 @@ - [Build and Test](#build-and-test) - [Integration Tests](#integration-tests) - [Linter](#linter) + - [Generate API Actions](#generate-api-actions) - [Submitting Changes](#submitting-changes) # Developer Guide @@ -85,6 +86,10 @@ rubocop -a rubocop --auto-gen-config ``` +### Generate API Actions + +All changes to the API actions should be done via the `api_generator`. For more information, see the [API Generator's USER_GUIDE](./api_generator/USER_GUIDE.md). + ## Submitting Changes See [CONTRIBUTING](CONTRIBUTING.md). diff --git a/api_generator/.rubocop.yml b/api_generator/.rubocop.yml new file mode 100644 index 000000000..72560a6c4 --- /dev/null +++ b/api_generator/.rubocop.yml @@ -0,0 +1,21 @@ +require: rubocop-rake +AllCops: + Include: + - 'lib/**/*.rb' + - 'Rakefile' + NewCops: enable + +Metrics/CyclomaticComplexity: + Enabled: false +Metrics/MethodLength: + Enabled: false +Metrics/AbcSize: + Enabled: false +Metrics/PerceivedComplexity: + Enabled: false + +Layout/EmptyLineAfterGuardClause: + Enabled: false + +Style/MultilineBlockChain: + Enabled: false diff --git a/api_generator/.ruby-version b/api_generator/.ruby-version new file mode 100644 index 000000000..fd2a01863 --- /dev/null +++ b/api_generator/.ruby-version @@ -0,0 +1 @@ +3.1.0 diff --git a/api_generator/USER_GUIDE.md b/api_generator/USER_GUIDE.md new file mode 100644 index 000000000..9ca757ce5 --- /dev/null +++ b/api_generator/USER_GUIDE.md @@ -0,0 +1,48 @@ +This API Generator generates API actions for the OpenSearch Ruby client based off of [OpenSearch OpenAPI specification](https://github.com/opensearch-project/opensearch-api-specification/blob/main/OpenSearch.openapi.json). All changes to the API actions should be done via the this generator. If you find an error in the API actions, it most likely comes from the spec. So, please submit a new issue to the [OpenSearch API specification](https://github.com/opensearch-project/opensearch-api-specification/issues/new/choose) repo first. + +--- + +### Usage +This generator should be run everytime the OpenSearch API Specification is updated to propagate the changes to the Ruby client. For now, this must be done manually: +- Create a new branch from `main` +- Download the latest OpenSearch API Specification from [The API Spec Repo](https://github.com/opensearch-project/opensearch-api-specification/blob/main/OpenSearch.openapi.json) +- Run the generator with the API Spec downloaded previously (see below) +- Run Rubocop with `-a` flag to remove redundant spacing from the generated code `rubocop -a` +- Commit and create a PR to merge the updated API actions into `main`. + +### Generate API Actions +Install all dependencies +```bash +bundle install +``` + +Import the API Generator and load the OpenSearch OpenAPI specification into a generator instance +```ruby +require './lib/api_generator' +generator = ApiGenerator.new('./OpenSearch.openapi.json') +``` + +The `generate` method accepts the path to the root directory of the `opensearch-ruby` gem as a parameter. By default, it points to the parent directory of the folder containing the generator script. For example to generate all actions into the `tmp` directory: +```ruby +generator.generate('./tmp') +``` + +You can also target a specific API version by passing in the version number as a parameter. For example to generate all actions for version `1.0` into the `tmp` directory: +```ruby +generator.generate(version: '1.0') +``` + +The generator also support incremental generation. For example, to generate all actions of the `cat` namespace: +```ruby +generator.generate(namespace: 'cat') +``` + +To limit it to specific actions of a namespace: +```ruby +generator.generate(namespace: 'cat', actions: %w[aliases allocation]) +``` + +Note that the root namespace is presented by an empty string `''`. For example, to generate all actions of the root namespace for OS version 2.3: +```ruby +generator.generate(version: '2.3', namespace: '') +``` diff --git a/api_generator/gemfile b/api_generator/gemfile new file mode 100644 index 000000000..d2e247092 --- /dev/null +++ b/api_generator/gemfile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'rake' +gem 'rubocop', '~> 1.44', require: false +gem 'rubocop-rake', require: false +gem 'openapi3_parser' +gem 'mustache', '~> 1' +gem 'awesome_print' +gem 'activesupport', '~> 7' diff --git a/api_generator/lib/action.rb b/api_generator/lib/action.rb new file mode 100644 index 000000000..fd1884b71 --- /dev/null +++ b/api_generator/lib/action.rb @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require_relative 'operation' +require_relative 'version' +require_relative 'parameter' + +# A collection of operations that comprise a single API Action +class Action + attr_reader :group, :name, :namespace, :http_verbs, :urls, :description, :external_docs, + :parameters, :path_params, :query_params, + :body, :body_description, :body_required + + # @param [Array] operations + def initialize(operations) + @operations = operations + @group = operations.first.group + @name = operations.first.action + @namespace = operations.first.namespace + @http_verbs = operations.map(&:http_verb).uniq + @urls = operations.map(&:url).uniq + @description = operations.map(&:description).find(&:present?) + @external_docs = operations.map(&:external_docs).find(&:present?) + @external_docs = nil if @external_docs == 'https://opensearch.org/docs/latest' + + dup_params = operations.flat_map(&:parameters) + @path_params = dup_params.select { |p| p.in == 'path' } + path_param_names = @path_params.map(&:name).to_set + @query_params = dup_params.select { |p| p.in == 'query' && !path_param_names.include?(p.name) } + @parameters = @path_params + @query_params + @parameters.each { |p| p.spec.node_data['required'] = p.name.in?(required_components) } + + @body = operations.map(&:request_body).find(&:present?) + @body_required = 'body'.in?(required_components) + @body_description = @body&.content&.[]('application/json')&.schema&.description if @body.present? + end + + # @return [Set] The names of input components that are required by the action. + # A component is considered required if it is required by all operations that make up the action. + def required_components + @required_components ||= @operations.map do |op| + set = Set.new(op.parameters.select(&:required?).map(&:name)) + set.add('body') if op.request_body&.required? + set + end.reduce(&:intersection) + end +end diff --git a/api_generator/lib/action_generator.rb b/api_generator/lib/action_generator.rb new file mode 100644 index 000000000..1b2685c4c --- /dev/null +++ b/api_generator/lib/action_generator.rb @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require_relative 'base_generator' +require_relative 'action' + +# Generate an API Action via Mustache +class ActionGenerator < BaseGenerator + self.template_file = './templates/action.mustache' + attr_reader :module_name, :method_name, :valid_params_constant_name, + :method_description, :argument_descriptions, :external_docs + + # Actions that use perform_request_simple_ignore_404 + SIMPLE_IGNORE_404 = %w[exists + indices.exists + indices.exists_alias + indices.exists_template + indices.exists_type].to_set.freeze + + # Actions that use perform_request_complex_ignore_404 + COMPLEX_IGNORE_404 = %w[delete + get + indices.flush_synced + indices.delete_template + indices.delete + security.get_role + security.get_user + snapshot.status + snapshot.get + snapshot.get_repository + snapshot.delete_repository + snapshot.delete + update + watcher.delete_watch].to_set.freeze + + # Actions that use perform_request_ping + PING = %w[ping].to_set.freeze + + # @param [Pathname] output_folder + # @param [Action] action + def initialize(output_folder, action) + super(output_folder) + @action = action + @urls = action.urls.map { |u| u.split('/').select(&:present?) }.uniq + @external_docs = action.external_docs + @module_name = action.namespace&.camelize + @method_name = action.name.underscore + @valid_params_constant_name = "#{action.name.upcase}_QUERY_PARAMS" + @method_description = action.description + @argument_descriptions = params_desc + [body_desc].compact + end + + def url_components + @urls.max_by(&:length) + .map { |e| e.starts_with?('{') ? "_#{e[/{(.+)}/, 1]}" : "'#{e}'" } + .join(', ') + end + + def http_verb + case @action.http_verbs.sort + when %w[get post] + 'body ? OpenSearch::API::HTTP_POST : OpenSearch::API::HTTP_GET' + when %w[post put] + diff_param = @urls.map(&:to_set).sort_by(&:size).reverse.reduce(&:difference).first + "_#{diff_param[/{(.+)}/, 1]} ? OpenSearch::API::HTTP_PUT : OpenSearch::API::HTTP_POST" + else + "OpenSearch::API::HTTP_#{@action.http_verbs.first.upcase}" + end + end + + def required_args + @action.required_components.map { |arg| { arg: } } + .tap { |args| args.last&.[]=('_blank_line', true) } + end + + def path_params + @action.path_params.map { |p| { name: p.name, listify: p.is_array } } + .tap { |args| args.last&.[]=('_blank_line', true) } + end + + def query_params + @action.query_params.map { |p| { name: p.name } } + end + + def listify_query_params + @action.query_params.select(&:is_array).map { |p| { name: p.name } } + .tap { |args| args.first&.[]=('_blank_line', true) } + end + + def perform_request + args = 'method, url, params, body, headers' + return "perform_request_simple_ignore_404(#{args})" if SIMPLE_IGNORE_404.include?(@action.group) + return "perform_request_complex_ignore_404(#{args}, arguments)" if COMPLEX_IGNORE_404.include?(@action.group) + return "perform_request_ping(#{args})" if PING.include?(@action.group) + "perform_request(#{args}).body" + end + + private + + def output_file + create_folder(*[@output_folder, @action.namespace].compact).join("#{@action.name}.rb") + end + + def params_desc + @action.parameters.map do |p| + { data_type: p.ruby_type, + name: p.name, + required: p.required?, + description: p.description, + default: p.default, + deprecated: p.deprecated? } + end + end + + def body_desc + return unless @action.body.present? + { data_type: :Hash, + name: :body, + description: @action.body_description, + required: @action.body_required } + end +end diff --git a/api_generator/lib/api_generator.rb b/api_generator/lib/api_generator.rb new file mode 100644 index 000000000..462201156 --- /dev/null +++ b/api_generator/lib/api_generator.rb @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require 'openapi3_parser' +require_relative 'action' +require_relative 'action_generator' +require_relative 'spec_generator' +require_relative 'namespace_generator' +require_relative 'index_generator' + +# Generate API endpoints for OpenSearch Ruby client +class ApiGenerator + HTTP_VERBS = %w[get post put patch delete patch head].freeze + EXISTING_NAMESPACES = Set.new(%w[ + cluster + nodes + indices + ingest + snapshot + tasks + cat + remote + dangling_indices + features + shutdown + ]).freeze + + # @param [String] openapi_spec location of the OpenSearch API spec file [required] + def initialize(openapi_spec) + @spec = Openapi3Parser.load_file(openapi_spec) + end + + # @param [String] gem_folder location of the API Gem folder (default to the parent folder of the generator) + # @param [String] version target OpenSearch version to generate like "2.5" or "3.0" + # @param [String] namespace namespace to generate (Default to all namespaces. Use '' for root) + # @param [Array] actions list of actions in the specified namespace to generate (Default to all actions) + def generate(gem_folder = '../', version: nil, namespace: nil, actions: nil) + gem_folder = Pathname gem_folder + namespaces = EXISTING_NAMESPACES.dup + target_actions(version, namespace, actions).each do |action| + ActionGenerator.new(gem_folder.join('lib/opensearch/api/actions'), action).generate + SpecGenerator.new(gem_folder.join('spec/opensearch/api/actions'), action).generate + NamespaceGenerator.new(gem_folder.join('lib/opensearch/api/namespace'), action.namespace).generate(namespaces) + end + IndexGenerator.new(gem_folder.join('lib/opensearch'), namespaces).generate + end + + private + + def target_actions(version, namespace, actions) + namespace = namespace.to_s + actions = Array(actions).map(&:to_s).to_set unless actions.nil? + + operations = @spec.paths.flat_map do |url, path| + path.to_h.slice(*HTTP_VERBS).compact.map do |verb, operation_spec| + operation = Operation.new operation_spec, url, verb + operation.part_of?(version, namespace, actions) ? operation : nil + end + end.compact + + operations.group_by(&:group).values.map { |ops| Action.new ops } + end +end diff --git a/api_generator/lib/base_generator.rb b/api_generator/lib/base_generator.rb new file mode 100644 index 000000000..dad8d78ff --- /dev/null +++ b/api_generator/lib/base_generator.rb @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require 'openapi3_parser' +require 'mustache' +require 'active_support/all' + +# Base Mustache Generator +class BaseGenerator < Mustache + self.template_path = './templates' + + def initialize(output_folder) + @output_folder = output_folder + super + end + + def license_header + Pathname('./templates/license_header.txt').read + end + + def generated_code_warning + "# This code was generated from OpenSearch API Spec.\n" \ + '# Update the code generation logic instead of modifying this file directly.' + end + + def generate + output_file.write(render) + end + + private + + def output_file + raise "'#{__method__}' Must be implemented by subclass" + end + + def create_folder(*components) + folder = components.reduce(&:+) + folder.mkpath unless folder.exist? + folder + end +end diff --git a/api_generator/lib/index_generator.rb b/api_generator/lib/index_generator.rb new file mode 100644 index 000000000..60d967db8 --- /dev/null +++ b/api_generator/lib/index_generator.rb @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require 'openapi3_parser' +require_relative 'base_generator' + +# Generate the index file via Mustache +class IndexGenerator < BaseGenerator + self.template_file = './templates/index.mustache' + + def initialize(output_folder, namespaces) + @namespaces = namespaces.compact + super(output_folder) + end + + def legacy_license_header + Pathname('./templates/legacy_license_header.txt').read + end + + def namespace_modules + modules = @namespaces.to_a.sort.map do |namespace| + { name: "OpenSearch::API::#{namespace.camelcase}", comma: ',' } + end + modules.last[:comma] = '' + modules + end + + private + + def output_file + create_folder(@output_folder).join('api.rb') + end +end diff --git a/api_generator/lib/namespace_generator.rb b/api_generator/lib/namespace_generator.rb new file mode 100644 index 000000000..55092486e --- /dev/null +++ b/api_generator/lib/namespace_generator.rb @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require_relative 'base_generator' + +# Generate a Namespace file via Mustache +class NamespaceGenerator < BaseGenerator + self.template_file = './templates/namespace.mustache' + attr_reader :namespace + + def initialize(output_folder, namespace) + super(output_folder) + @namespace = namespace + end + + def module_name + @namespace.camelize + end + + def client_name + "#{@namespace.camelize}Client" + end + + def generate(existing_namespaces) + return if @namespace.nil? || @namespace.in?(existing_namespaces) + existing_namespaces.add(@namespace) + super() + end + + private + + def output_file + create_folder(@output_folder).join("#{@namespace}.rb") + end +end diff --git a/api_generator/lib/operation.rb b/api_generator/lib/operation.rb new file mode 100644 index 000000000..025af5231 --- /dev/null +++ b/api_generator/lib/operation.rb @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require 'openapi3_parser' +require 'openapi3_parser/node/operation' +require_relative 'version' +require_relative 'parameter' + +# Wrapper for Openapi3Parser::Node::Operation that adds extra info unique to OpenSearch +class Operation < Openapi3Parser::Node::Operation + attr_reader :url, :http_verb, :group, :action, :namespace, + :version_added, :version_removed, :version_deprecated, :external_docs + + # @param [Openapi3Parser::Node::Operation] spec Operation Spec + # @param [String] url + # @param [String] http_verb + def initialize(spec, url, http_verb) + super(spec.node_data, spec.node_context) + @url = url + @http_verb = http_verb + @group = spec['x-operation-group'] + @action, @namespace = @group.split('.').reverse + @version_added = Version.new(spec['x-version-added'] || '0.0.0') + @version_removed = Version.new(spec['x-version-removed'] || '999.999.999') + @version_deprecated = Version.new spec['x-version-deprecated'] + @external_docs = spec['externalDocs']&.[]('url') + end + + # @return [Array] collection of path and query parameters + def parameters + @parameters ||= super.map { |p| Parameter.new(p) } + end + + # @param [String] version is the operation part of this version? + # @param [String, NilClass] namespace is the operation part of this namespace? + # @param [Set, NilClass] actions is the operation part of any of these actions? + def part_of?(version, namespace, actions) + version = Version.new(version) + part_of_version = version.nil? || (version_added <= version && version < version_removed) + part_of_namespace = namespace.nil? || @namespace == namespace || (@namespace.nil? && namespace == '') + part_of_actions = actions.nil? || actions.include?(action) + part_of_version && part_of_namespace && part_of_actions + end +end diff --git a/api_generator/lib/parameter.rb b/api_generator/lib/parameter.rb new file mode 100644 index 000000000..696f5fc35 --- /dev/null +++ b/api_generator/lib/parameter.rb @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require 'openapi3_parser/node/parameter' +require_relative 'version' + +# Wrapper for Openapi3Parser::Node::Parameter that adds extra info unique to OpenSearch +class Parameter < Openapi3Parser::Node::Parameter + attr_reader :spec, :type, :ruby_type, :is_array, :default, :deprecated + + # @param [Openapi3Parser::Node::Parameter] spec Parameter Spec + def initialize(spec) + super(spec.node_data, spec.node_context) + @spec = spec + @type = schema&.[]('x-data-type') || schema&.type + @ruby_type = @type.capitalize + @is_array = schema&.type == 'array' + @default = schema&.default + @deprecated = schema&.deprecated? == true + end + + # @return [any] example value for this parameter + def example_value + return 'songs' if type == 'string' + return 42 if type == 'integer' + return true if type == 'boolean' + return %w[books movies] if type == 'array' + return '1m' if type == 'time' + raise "Unknown type #{type}" + end + + # @return [String] value to be interpolated into the url path to be passed to the transport layer + def expected_path_value + type == 'array' ? example_value.join(',') : example_value.to_s + end + + # @return [any] query value to be passed to the transport layer + def expected_query_value + return "'#{example_value}'" if type.in?(%w[string time]) + return "'#{example_value.join(',')}'" if type == 'array' + example_value + end + + # @return [any] value to be passed to the client in the spec + def client_double_value + return "'#{example_value}'" if type.in?(%w[string time]) + return "%w[#{example_value.join(' ')}]" if type == 'array' + example_value + end +end diff --git a/api_generator/lib/spec_generator.rb b/api_generator/lib/spec_generator.rb new file mode 100644 index 000000000..a772ed2f3 --- /dev/null +++ b/api_generator/lib/spec_generator.rb @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +require_relative 'base_generator' +require_relative 'action' + +# Generate Spec test for an API Action via Mustache +class SpecGenerator < BaseGenerator + self.template_file = './templates/spec.mustache' + attr_reader :action, :http_verb + + delegate :namespace, :name, to: :action + + # @param [Pathname] output_folder + # @param [Action] action + def initialize(output_folder, action) + super(output_folder) + @action = action + @http_verb = action.http_verbs.min.upcase + end + + def expected_url_path + action.urls.max_by(&:length).split('/').select(&:present?).map do |component| + next component unless component.start_with?('{') + param = action.path_params.find { |p| p.name == component[/{(.+)}/, 1] } + param.expected_path_value + end.join('/') + end + + def expected_query_params + action.query_params.map do |p| + { pre: ' ', + key: p.name, + value: p.expected_query_value, + post: ',' } + end.tap do |params| + params.first&.update(pre: '{ ') + params.last&.update(post: ' },') + end + end + + def body + return '{}' if action.required_components.include?('body') + return 'nil' if action.body.nil? + http_verb.in?(%w[PUT POST PATCH]) ? '{}' : 'nil' + end + + def required_components + action.required_components.map do |component| + { arg: component, + others: other_required_components(component) } + end.tap do |components| + components.last&.update(blank_line: true) + end + end + + def client_double_args + args = (action.path_params + action.query_params).map { |p| { key: p.name, value: p.client_double_value } } + args += [{ key: 'body', value: '{}' }] unless body == 'nil' + args.last&.update(last: true) + args + end + + private + + def other_required_components(component) + others = action.required_components.reject { |c| c == component }.map do |c| + "#{c}: #{arg_value(c)}" + end.join(', ') + "(#{others})" unless others.empty? + end + + def arg_value(component) + return body if component == 'body' + action.path_params.find { |p| p.name == component }&.client_double_value + end + + def output_file + create_folder(*[@output_folder, @action.namespace].compact).join("#{@action.name}_spec.rb") + end +end diff --git a/api_generator/lib/version.rb b/api_generator/lib/version.rb new file mode 100644 index 000000000..1d943d5e9 --- /dev/null +++ b/api_generator/lib/version.rb @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# frozen_string_literal: true + +# OpenSearch Version Number +class Version + include Comparable + attr_reader :numbers + + # @param [String, NilClass] version_str + def initialize(version_str) + @version_str = version_str + @numbers = version_str&.split('.')&.map(&:to_i) + end + + # @param [Version] other + def <=>(other) + numbers.zip(other.numbers).each do |self_, other_| + return 1 if self_ > other_ + return -1 if self_ < other_ + end + 0 + end + + def nil? + numbers.nil? + end +end diff --git a/api_generator/templates/action.module.mustache b/api_generator/templates/action.module.mustache new file mode 100644 index 000000000..bba8a75b5 --- /dev/null +++ b/api_generator/templates/action.module.mustache @@ -0,0 +1,50 @@ +module Actions + {{valid_params_constant_name}} = Set.new(%i[ + {{#query_params}} + {{name}} + {{/query_params}} + ]).freeze + + # {{{method_description}}} + # + {{#argument_descriptions}} + # @option arguments [{{data_type}}] :{{name}} {{#required}}*Required* {{/required}}{{#deprecated}}DEPRECATED {{/deprecated}}{{#default}}(default: {{default}}) {{/default}}{{description}} + {{/argument_descriptions}} + {{#external_docs}} + # + # {API Reference}[{{{external_docs}}}] + {{/external_docs}} + def {{method_name}}(arguments = {}) + {{#required_args}} + raise ArgumentError, "Required argument '{{arg}}' missing" unless arguments[:{{arg}}] + {{#_blank_line}} + + {{/_blank_line}} + {{/required_args}} + arguments = arguments.clone + {{#path_params}} + {{#listify}} + _{{name}} = Utils.__listify(arguments.delete(:{{name}})) + {{/listify}} + {{^listify}} + _{{name}} = arguments.delete(:{{name}}) + {{/listify}} + {{#_blank_line}} + + {{/_blank_line}} + {{/path_params}} + headers = arguments.delete(:headers) || {} + body = arguments.delete(:body) + url = Utils.__pathify {{{url_components}}} + method = {{{http_verb}}} + params = Utils.__validate_and_extract_params arguments, {{valid_params_constant_name}} + {{#listify_query_params}} + {{#_blank_line}} + + {{/_blank_line}} + params[:{{name}}] = Utils.__listify(params[:{{name}}]) if params[:{{name}}] + {{/listify_query_params}} + + {{{perform_request}}} + end +end diff --git a/api_generator/templates/action.mustache b/api_generator/templates/action.mustache new file mode 100644 index 000000000..97fb8e995 --- /dev/null +++ b/api_generator/templates/action.mustache @@ -0,0 +1,17 @@ +{{{license_header}}} +{{{generated_code_warning}}} + +# frozen_string_literal: true + +module OpenSearch + module API + {{#module_name}} + module {{module_name}} + {{>action.module}} + end + {{/module_name}} + {{^module_name}} + {{>action.module}} + {{/module_name}} + end +end diff --git a/api_generator/templates/index.mustache b/api_generator/templates/index.mustache new file mode 100644 index 000000000..40e119312 --- /dev/null +++ b/api_generator/templates/index.mustache @@ -0,0 +1,67 @@ +{{{legacy_license_header}}} +{{{generated_code_warning}}} + +# frozen_string_literal: true + +require 'opensearch/api/namespace/common' +require 'opensearch/api/utils' +require 'opensearch/api/actions/params_registry' + +Dir[File.expand_path('api/actions/**/params_registry.rb', __dir__)].sort.each { |f| require f } +Dir[File.expand_path('api/actions/**/*.rb', __dir__)].sort.each { |f| require f } +Dir[File.expand_path('api/namespace/**/*.rb', __dir__)].sort.each { |f| require f } + +module OpenSearch + module API + DEFAULT_SERIALIZER = MultiJson + + COMMON_PARAMS = [ + :ignore, # Client specific parameters + :index, :id, # :index/:id + :body, # Request body + :node_id, # Cluster + :name, # Alias, template, settings, warmer, ... + :field # Get field mapping + ] + + COMMON_QUERY_PARAMS = [ + :ignore, # Client specific parameters + :format, # Search, Cat, ... + :pretty, # Pretty-print the response + :human, # Return numeric values in human readable format + :filter_path, # Filter the JSON response + :opaque_id # Use X-Opaque-Id + ] + + HTTP_GET = 'GET' + HTTP_HEAD = 'HEAD' + HTTP_PATCH = 'PATCH' + HTTP_POST = 'POST' + HTTP_PUT = 'PUT' + HTTP_DELETE = 'DELETE' + + UNDERSCORE_SEARCH = '_search' + UNDERSCORE_ALL = '_all' + DEFAULT_DOC = '_doc' + + # Auto-include all namespaces in the receiver + def self.included(base) + base.send :include, + OpenSearch::API::Common, + OpenSearch::API::Actions, + {{#namespace_modules}} + {{name}}{{comma}} + {{/namespace_modules}} + end + + # The serializer class + def self.serializer + settings[:serializer] || DEFAULT_SERIALIZER + end + + # Access the module settings + def self.settings + @settings ||= {} + end + end +end diff --git a/api_generator/templates/legacy_license_header.txt b/api_generator/templates/legacy_license_header.txt new file mode 100644 index 000000000..62dbc249f --- /dev/null +++ b/api_generator/templates/legacy_license_header.txt @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/api_generator/templates/license_header.txt b/api_generator/templates/license_header.txt new file mode 100644 index 000000000..ff4fd04d1 --- /dev/null +++ b/api_generator/templates/license_header.txt @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. diff --git a/api_generator/templates/namespace.mustache b/api_generator/templates/namespace.mustache new file mode 100644 index 000000000..10b6e33a8 --- /dev/null +++ b/api_generator/templates/namespace.mustache @@ -0,0 +1,24 @@ +{{{license_header}}} +{{{generated_code_warning}}} + +# frozen_string_literal: true + +module OpenSearch + module API + module {{module_name}} + module Actions; end + + # Client for the "{{namespace}}" namespace (includes the {{module_name}}::Actions methods) + class {{client_name}} + include {{module_name}}::Actions + include Common::Client + include Common::Client::Base + end + + # Proxy method for {{client_name}}, available in the receiving object + def {{namespace}} + @{{namespace}} ||= {{client_name}}.new(self) + end + end + end +end diff --git a/api_generator/templates/spec.mustache b/api_generator/templates/spec.mustache new file mode 100644 index 000000000..ca88a0ea0 --- /dev/null +++ b/api_generator/templates/spec.mustache @@ -0,0 +1,42 @@ +{{{license_header}}} +{{{generated_code_warning}}} + +# frozen_string_literal: true + +require_relative '{{#namespace}}../{{/namespace}}../../../spec_helper' + +describe 'client{{#namespace}}.{{namespace}}{{/namespace}}#{{name}}' do + let(:expected_args) do + [ + '{{http_verb}}', + '{{expected_url_path}}', + {{#expected_query_params}} + {{pre}}{{key}}: {{{value}}}{{post}} + {{/expected_query_params}} + {{body}}, + {} + ] + end + + let(:client) do + Class.new { include OpenSearch::API }.new + end + {{#required_components}} + + it 'requires the :{{arg}} argument' do + expect do + client{{#namespace}}.{{namespace}}{{/namespace}}.{{name}}{{{others}}} + end.to raise_exception(ArgumentError) + end + {{#blank_line}} + + {{/blank_line}} + {{/required_components}} + it 'performs the request with all optional params' do + expect(client_double{{#namespace}}.{{namespace}}{{/namespace}}.{{name}}( + {{#client_double_args}} + {{key}}: {{{value}}}{{^last}},{{/last}} + {{/client_double_args}} + )).to eq({}) + end +end diff --git a/lib/opensearch/api/namespace/common.rb b/lib/opensearch/api/namespace/common.rb index 510b420e2..5ed25b078 100644 --- a/lib/opensearch/api/namespace/common.rb +++ b/lib/opensearch/api/namespace/common.rb @@ -45,6 +45,27 @@ def initialize(client) def perform_request(method, path, params = {}, body = nil, headers = nil) client.perform_request method, path, params, body, headers end + + def perform_request_simple_ignore404(method, path, params, body, headers) + Utils.__rescue_from_not_found do + perform_request(method, path, params, body, headers).status == 200 + end + end + + def perform_request_complex_ignore404(method, path, params, body, headers, arguments) + if Array(arguments[:ignore]).include?(404) + Utils.__rescue_from_not_found { perform_request(method, path, params, body, headers).body } + else + perform_request(method, path, params, body, headers).body + end + end + + def perform_request_ping(method, path, params, body, headers) + perform_request(method, path, params, body, headers).status == 200 + rescue StandardError => e + raise e unless e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i + false + end end end end From aaa065d154368591ab9831b4052ce6a943a19f73 Mon Sep 17 00:00:00 2001 From: Theo Truong Date: Wed, 9 Aug 2023 12:54:55 -0600 Subject: [PATCH 2/2] # Took advantage of the provided gem folder to grep for folders representing namespaces. We don't need to hardcode existing namespaces anymore! Signed-off-by: Theo Truong --- api_generator/lib/api_generator.rb | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/api_generator/lib/api_generator.rb b/api_generator/lib/api_generator.rb index 462201156..843cc64aa 100644 --- a/api_generator/lib/api_generator.rb +++ b/api_generator/lib/api_generator.rb @@ -16,19 +16,6 @@ # Generate API endpoints for OpenSearch Ruby client class ApiGenerator HTTP_VERBS = %w[get post put patch delete patch head].freeze - EXISTING_NAMESPACES = Set.new(%w[ - cluster - nodes - indices - ingest - snapshot - tasks - cat - remote - dangling_indices - features - shutdown - ]).freeze # @param [String] openapi_spec location of the OpenSearch API spec file [required] def initialize(openapi_spec) @@ -41,7 +28,7 @@ def initialize(openapi_spec) # @param [Array] actions list of actions in the specified namespace to generate (Default to all actions) def generate(gem_folder = '../', version: nil, namespace: nil, actions: nil) gem_folder = Pathname gem_folder - namespaces = EXISTING_NAMESPACES.dup + namespaces = existing_namespaces(gem_folder) target_actions(version, namespace, actions).each do |action| ActionGenerator.new(gem_folder.join('lib/opensearch/api/actions'), action).generate SpecGenerator.new(gem_folder.join('spec/opensearch/api/actions'), action).generate @@ -65,4 +52,8 @@ def target_actions(version, namespace, actions) operations.group_by(&:group).values.map { |ops| Action.new ops } end + + def existing_namespaces(gem_folder) + gem_folder.join('lib/opensearch/api/actions').children.select(&:directory?).map(&:basename).map(&:to_s).to_set + end end