From 3b879015627d35411b9163666970a4ebb0ac0e27 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 24 Jun 2024 13:28:49 -0400 Subject: [PATCH 001/120] Store position in export --- app/assets/javascripts/components/export_case/_modal.html | 2 +- app/assets/javascripts/services/caseCSVSvc.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/assets/javascripts/components/export_case/_modal.html b/app/assets/javascripts/components/export_case/_modal.html index 87c50f1d6..48ab1964a 100644 --- a/app/assets/javascripts/components/export_case/_modal.html +++ b/app/assets/javascripts/components/export_case/_modal.html @@ -29,7 +29,7 @@

Export Case: {{ ctrl.theCase.ca Detailed export is only supported from the individual Case view.

- CSV file with Team Name,Case Name,Case ID,Query Text,Doc ID,Title,Rating,Field1,...,FieldN where Field1,...,FieldN are specified under Settings in the Displayed Fields field. + CSV file with Team Name,Case Name,Case ID,Query Text,Doc ID,Position,Title,Rating,Field1,...,FieldN where Field1,...,FieldN are specified under Settings in the Displayed Fields field. diff --git a/app/assets/javascripts/services/caseCSVSvc.js b/app/assets/javascripts/services/caseCSVSvc.js index 871281b29..23f16e74e 100644 --- a/app/assets/javascripts/services/caseCSVSvc.js +++ b/app/assets/javascripts/services/caseCSVSvc.js @@ -79,6 +79,7 @@ 'Case ID', 'Query Text', 'Doc ID', + 'Doc Position', 'Title', 'Rating', ]; @@ -353,7 +354,7 @@ csvContent += dataString + EOL; } else { - angular.forEach(docs, function (doc) { + angular.forEach(docs, function (doc, index) { var dataString; var infoArray = []; @@ -362,6 +363,7 @@ infoArray.push(stringifyField(aCase.lastScore.case_id)); infoArray.push(stringifyField(query.queryText)); infoArray.push(stringifyField(doc.id)); + infoArray.push(stringifyField(index+1)); infoArray.push(stringifyField(doc.title)); infoArray.push(stringifyField(doc.getRating())); From 1a834ff15f4371e4a225c41fc80007e0ba7630e9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 24 Jun 2024 13:29:10 -0400 Subject: [PATCH 002/120] nicer message --- .../admin/websocket_tester/_notification.turbo_stream.erb | 2 +- app/views/admin/websocket_tester/index.html.erb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/views/admin/websocket_tester/_notification.turbo_stream.erb b/app/views/admin/websocket_tester/_notification.turbo_stream.erb index c05275892..111576b75 100644 --- a/app/views/admin/websocket_tester/_notification.turbo_stream.erb +++ b/app/views/admin/websocket_tester/_notification.turbo_stream.erb @@ -7,7 +7,7 @@ <% else %> <% end %> diff --git a/app/views/admin/websocket_tester/index.html.erb b/app/views/admin/websocket_tester/index.html.erb index 965731774..610347924 100644 --- a/app/views/admin/websocket_tester/index.html.erb +++ b/app/views/admin/websocket_tester/index.html.erb @@ -4,6 +4,9 @@

This tool is for testing out websocket handling in Quepid.

+

+ After clicking the button you should see a count down counter that updates every second in a alert box. +

From 9765a8f4ab54d0528d78bf3172549976b24210ab Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 24 Jun 2024 13:29:16 -0400 Subject: [PATCH 003/120] better temp data --- lib/tasks/sample_data.thor | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tasks/sample_data.thor b/lib/tasks/sample_data.thor index 7cfc88973..d5b4994d3 100644 --- a/lib/tasks/sample_data.thor +++ b/lib/tasks/sample_data.thor @@ -227,6 +227,7 @@ class SampleData < Thor osc.members << osc_member_user unless osc.members.include?(osc_member_user) osc.members << realistic_activity_user unless osc.members.include?(realistic_activity_user) osc.cases << tens_of_queries_case unless osc.members.include?(tens_of_queries_case) + osc.search_endpoints << statedecoded_solr_endpoint unless osc.search_endpoints.include?(statedecoded_solr_endpoint) print_step 'End of seeding teams................' # Books From 49267a1bd8e7aab8909d171e945622b4df24e4ab Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 24 Jun 2024 13:29:28 -0400 Subject: [PATCH 004/120] remove debugging --- app/views/layouts/application.html.erb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 021129ce6..0924b4ddc 100755 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -69,11 +69,7 @@ <%= render 'layouts/header' %> - +
From f916fb1cbe3e51a61d74aca08c4bbb939d9623ba Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 26 Jun 2024 13:27:19 -0400 Subject: [PATCH 005/120] started the query_runner --- .../admin/query_runner_controller.rb | 18 ++ app/controllers/proxy_controller.rb | 11 -- app/jobs/query_runner_job.rb | 130 ++++++++++++++ app/services/fetch_service.rb | 169 ++++++++++++++++++ app/views/admin/home/index.html.erb | 1 + .../_notification.turbo_stream.erb | 17 ++ app/views/admin/query_runner/index.html.erb | 30 ++++ config/routes.rb | 3 + 8 files changed, 368 insertions(+), 11 deletions(-) create mode 100644 app/controllers/admin/query_runner_controller.rb create mode 100644 app/jobs/query_runner_job.rb create mode 100644 app/services/fetch_service.rb create mode 100644 app/views/admin/query_runner/_notification.turbo_stream.erb create mode 100644 app/views/admin/query_runner/index.html.erb diff --git a/app/controllers/admin/query_runner_controller.rb b/app/controllers/admin/query_runner_controller.rb new file mode 100644 index 000000000..f514d0055 --- /dev/null +++ b/app/controllers/admin/query_runner_controller.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Admin + class QueryRunnerController < ApplicationController + before_action :set_case, only: [ :run_queries ] + + def index + + end + + def run_queries + @case = Case.find(params["case_id"]) # any case is accessible! + @try = @case.tries.where(try_number: params["try_number"]).first + QueryRunnerJob.perform_later @case, @try + redirect_to admin_query_runner_index_path, notice: "Query Runner Job was queued up for case #{@case.case_name} and try #{@try.name}." + end + end +end diff --git a/app/controllers/proxy_controller.rb b/app/controllers/proxy_controller.rb index cbf5eb1b5..2397eed43 100644 --- a/app/controllers/proxy_controller.rb +++ b/app/controllers/proxy_controller.rb @@ -112,17 +112,6 @@ def proxy_url_params params.require(:url) end - private - - def rack_header? header_name - predefined_rack_headers = %w[ - HTTP_VERSION HTTP_ACCEPT HTTP_ACCEPT_CHARSET HTTP_ACCEPT_ENCODING - HTTP_ACCEPT_LANGUAGE HTTP_CACHE_CONTROL HTTP_CONNECTION HTTP_HOST - HTTP_REFERER HTTP_USER_AGENT HTTP_X_REQUEST_ID - ] - - predefined_rack_headers.include?(header_name) - end end # rubocop:enable Layout/LineLength # rubocop:enable Metrics/AbcSize diff --git a/app/jobs/query_runner_job.rb b/app/jobs/query_runner_job.rb new file mode 100644 index 000000000..9d49eed01 --- /dev/null +++ b/app/jobs/query_runner_job.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +require 'action_controller' +require 'faraday' +require 'faraday/follow_redirects' + +class QueryRunnerJob < ApplicationJob + queue_as :bulk_processing + + def perform acase, try + + query_count = acase.queries.count + + acase.queries.each_with_index do |query, counter| + Turbo::StreamsChannel.broadcast_render_to( + :notifications, + target: 'notifications', + partial: 'admin/query_runner/notification', + locals: { query: query, query_count: query_count, counter: counter } + ) + + + + sleep(2) + end + + Turbo::StreamsChannel.broadcast_render_to( + :notifications, + target: 'notifications', + partial: 'admin/query_runner/notification', + locals: { query: nil, query_count: query_count, counter: 0 } + ) + + + end + + + def fetch + excluded_keys = [ :url, :action, :controller, :proxy_debug ] + + url_param = proxy_url_params + + proxy_debug = 'true' == params[:proxy_debug] + + uri = URI.parse(url_param) + url_without_path = "#{uri.scheme}://#{uri.host}" + url_without_path += ":#{uri.port}" unless uri.port.nil? + + connection = Faraday.new(url: url_without_path) do |faraday| + # Configure the connection options, such as headers or middleware + faraday.response :follow_redirects + faraday.response :logger, nil, { headers: proxy_debug, bodies: proxy_debug, errors: !Rails.env.test? } + faraday.ssl.verify = false + faraday.request :url_encoded + + matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } + + matching_headers.each do |name, value| + converted_name = name.sub('HTTP_', '') + converted_name = converted_name.tr('_', '-') + faraday.headers[converted_name] = value + end + + faraday.headers['Content-Type'] = 'application/json' + has_credentials = !uri.userinfo.nil? + if has_credentials + username, password = uri.userinfo.split(':') + faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" + end + faraday.adapter Faraday.default_adapter + end + + begin + if request.get? + response = connection.get do |req| + req.path = uri.path + query_params = request.query_parameters.except(*excluded_keys) + body_params = request.request_parameters.except(*query_params.keys) + + query_params.each do |param| + req.params[param.first] = param.second + end + + # the url parameter often has a format like + # http://myserver.com/search?query=text, and when this is passed in + # we get http://localhost:3000/proxy/fetch?url=http://myserver.com/search?query=text&rows=10 + # which means the parameter "query=text" is lost because the URL is parsed and this part is dropped, + # so here we add this one parameter back in if we have it. + if url_param.include?('?') + # sometimes our url looks like http://myserver.com/search?q=tiger + # But it could also be http://myserver.com/search?q=tiger? and that needs handling via the special .split + extra_query_param = url_param.split('?', 2).last.split('=') + + req.params[extra_query_param.first] = extra_query_param.second + end + unless body_params.empty? + + json_query = body_params.first.first + req.body = json_query + end + end + elsif request.post? + response = connection.post do |req| + req.path = uri.path + query_params = request.query_parameters.except(*excluded_keys) + body_params = request.request_parameters.except(*query_params.keys) # not sure about this and the request.raw_post + query_params.each do |param| + req.params[param.first] = param.second + end + unless body_params.empty? + json_query = request.raw_post + + req.body = json_query + end + end + end + + begin + data = JSON.parse(response.body) + return {json: data, status: response.status} + rescue JSON::ParserError + # sometimes the API is returning plain old text, like a "Unauthorized" type message. + return {json: {response: response.body}, status: response.status} + end + rescue Faraday::ConnectionFailed => e + return {json: { proxy_error: e.message }, status: :internal_server_error} + end + end + +end diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb new file mode 100644 index 000000000..29953a292 --- /dev/null +++ b/app/services/fetch_service.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +class FetchService + # include ProgressIndicator + + attr_reader :logger, :options + + def initialize url,proxy_debug, opts = {} + default_options = { + logger: Rails.logger + } + + @options = default_options.merge(opts.deep_symbolize_keys) + + @url = url + @proxy_debug = proxy_debug + @logger = @options[:logger] + end + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/PerceivedComplexity + def validate + params_to_use = @data_to_process + scorer_name = params_to_use[:scorer][:name] + scorer = Scorer.find_by(name: scorer_name) + if scorer.nil? + @book.errors.add(:scorer, "with name '#{scorer_name}' needs to be migrated over first.") + else + @book.scorer = scorer + end + + selection_strategy_name = params_to_use[:selection_strategy][:name] + selection_strategy = SelectionStrategy.find_by(name: selection_strategy_name) + if selection_strategy.nil? + @book.errors.add(:selection_strategy, + "Selection strategy with name '#{selection_strategy_name}' needs to be migrated over first.") + else + @book.selection_strategy = selection_strategy + end + + if params_to_use[:query_doc_pairs] + list_of_emails_of_users = [] + params_to_use[:query_doc_pairs].each do |query_doc_pair| + next unless query_doc_pair[:judgements] + + query_doc_pair[:judgements].each do |judgement| + list_of_emails_of_users << judgement[:user_email] if judgement[:user_email].present? + end + end + list_of_emails_of_users.uniq! + list_of_emails_of_users.each do |email| + unless User.exists?(email: email) + if options[:force_create_users] + User.invite!({ email: email, password: '', skip_invitation: true }, @current_user) + else + @book.errors.add(:base, "User with email '#{email}' needs to be migrated over first.") + end + end + end + end + end + + def fetch + excluded_keys = [ :url, :action, :controller, :proxy_debug ] + + url_param = @url + + proxy_debug = 'true' == @proxy_debug + + uri = URI.parse(url_param) + url_without_path = "#{uri.scheme}://#{uri.host}" + url_without_path += ":#{uri.port}" unless uri.port.nil? + + connection = Faraday.new(url: url_without_path) do |faraday| + # Configure the connection options, such as headers or middleware + faraday.response :follow_redirects + faraday.response :logger, nil, { headers: proxy_debug, bodies: proxy_debug, errors: !Rails.env.test? } + faraday.ssl.verify = false + faraday.request :url_encoded + + matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } + + matching_headers.each do |name, value| + converted_name = name.sub('HTTP_', '') + converted_name = converted_name.tr('_', '-') + faraday.headers[converted_name] = value + end + + faraday.headers['Content-Type'] = 'application/json' + has_credentials = !uri.userinfo.nil? + if has_credentials + username, password = uri.userinfo.split(':') + faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" + end + faraday.adapter Faraday.default_adapter + end + + begin + if request.get? + response = connection.get do |req| + req.path = uri.path + query_params = request.query_parameters.except(*excluded_keys) + body_params = request.request_parameters.except(*query_params.keys) + + query_params.each do |param| + req.params[param.first] = param.second + end + + # the url parameter often has a format like + # http://myserver.com/search?query=text, and when this is passed in + # we get http://localhost:3000/proxy/fetch?url=http://myserver.com/search?query=text&rows=10 + # which means the parameter "query=text" is lost because the URL is parsed and this part is dropped, + # so here we add this one parameter back in if we have it. + if url_param.include?('?') + # sometimes our url looks like http://myserver.com/search?q=tiger + # But it could also be http://myserver.com/search?q=tiger? and that needs handling via the special .split + extra_query_param = url_param.split('?', 2).last.split('=') + + req.params[extra_query_param.first] = extra_query_param.second + end + unless body_params.empty? + + json_query = body_params.first.first + req.body = json_query + end + end + elsif request.post? + response = connection.post do |req| + req.path = uri.path + query_params = request.query_parameters.except(*excluded_keys) + body_params = request.request_parameters.except(*query_params.keys) # not sure about this and the request.raw_post + query_params.each do |param| + req.params[param.first] = param.second + end + unless body_params.empty? + json_query = request.raw_post + + req.body = json_query + end + end + end + + begin + data = JSON.parse(response.body) + render json: data, status: response.status + rescue JSON::ParserError + # sometimes the API is returning plain old text, like a "Unauthorized" type message. + render plain: response.body, status: response.status + end + rescue Faraday::ConnectionFailed => e + render json: { proxy_error: e.message }, status: :internal_server_error + end + end + + + private + + def rack_header? header_name + predefined_rack_headers = %w[ + HTTP_VERSION HTTP_ACCEPT HTTP_ACCEPT_CHARSET HTTP_ACCEPT_ENCODING + HTTP_ACCEPT_LANGUAGE HTTP_CACHE_CONTROL HTTP_CONNECTION HTTP_HOST + HTTP_REFERER HTTP_USER_AGENT HTTP_X_REQUEST_ID + ] + + predefined_rack_headers.include?(header_name) + end +end diff --git a/app/views/admin/home/index.html.erb b/app/views/admin/home/index.html.erb index 5331d2768..6e5a9f024 100644 --- a/app/views/admin/home/index.html.erb +++ b/app/views/admin/home/index.html.erb @@ -25,6 +25,7 @@
  • <%= link_to 'Job Manager', sidekiq_web_path %>
    • +
  • <%= link_to 'Query Runner', admin_query_runner_index_path %>
  • <%= link_to 'Websocket Tester', admin_websocket_tester_index_path %>
diff --git a/app/views/admin/query_runner/_notification.turbo_stream.erb b/app/views/admin/query_runner/_notification.turbo_stream.erb new file mode 100644 index 000000000..807dc83f6 --- /dev/null +++ b/app/views/admin/query_runner/_notification.turbo_stream.erb @@ -0,0 +1,17 @@ +<%= turbo_stream.replace "notification" do %> + + <% unless query.nil? %> + Query: <%=query.query_text%> + <% end %> + <% if counter > 0 %> + + <% else %> + + <% end %> + +<% end %> diff --git a/app/views/admin/query_runner/index.html.erb b/app/views/admin/query_runner/index.html.erb new file mode 100644 index 000000000..b8ddc28b9 --- /dev/null +++ b/app/views/admin/query_runner/index.html.erb @@ -0,0 +1,30 @@ +<%= turbo_stream_from(:notifications) %> + +

Query Runner

+

+ This tool is for testing out background query running in Quepid. +

+

+ After clicking the button you should see a set of updates in a alert box. +

+ + + + + +<%= form_with(url: run_queries_admin_query_runner_index_path, local: true) do |form| %> +
+ <%= form.label :case_id %> + <%= form.text_field :case_id %> +
+ +
+ <%= form.label :try_number %> + <%= form.text_field :try_number %> +
+ +

+

+ <%= form.submit "Submit" %> +
+<% end %> diff --git a/config/routes.rb b/config/routes.rb index fb6170803..0f9fc8ef0 100755 --- a/config/routes.rb +++ b/config/routes.rb @@ -116,6 +116,9 @@ resources :websocket_tester, only: [ :index ] do post 'test_background_job', on: :collection end + resources :query_runner, only: [ :index ] do + post 'run_queries', on: :collection + end end # preview routes for mailers From ae6ba9f77094ceee5b1a02bd670008afa56be9e0 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 5 Jul 2024 14:34:09 +0200 Subject: [PATCH 006/120] good progress on TDD based approach to the fetch service. --- .../admin/query_runner_controller.rb | 10 +- app/controllers/proxy_controller.rb | 11 + app/jobs/query_runner_job.rb | 34 ++- app/services/fetch_service.rb | 267 +++++++++--------- app/services/snapshot_manager.rb | 4 +- config/routes.rb | 2 +- test/fixtures/search_endpoints.yml | 2 - test/lib/solr_arg_parser_test.rb | 2 + test/services/fetch_service_test.rb | 126 +++++++++ test/support/webmock.rb | 29 ++ 10 files changed, 328 insertions(+), 159 deletions(-) create mode 100644 test/services/fetch_service_test.rb diff --git a/app/controllers/admin/query_runner_controller.rb b/app/controllers/admin/query_runner_controller.rb index f514d0055..b1bf9a0c6 100644 --- a/app/controllers/admin/query_runner_controller.rb +++ b/app/controllers/admin/query_runner_controller.rb @@ -3,16 +3,16 @@ module Admin class QueryRunnerController < ApplicationController before_action :set_case, only: [ :run_queries ] - + def index - end def run_queries - @case = Case.find(params["case_id"]) # any case is accessible! - @try = @case.tries.where(try_number: params["try_number"]).first + @case = Case.find(params['case_id']) # any case is accessible! + @try = @case.tries.where(try_number: params['try_number']).first QueryRunnerJob.perform_later @case, @try - redirect_to admin_query_runner_index_path, notice: "Query Runner Job was queued up for case #{@case.case_name} and try #{@try.name}." + redirect_to admin_query_runner_index_path, + notice: "Query Runner Job was queued up for case #{@case.case_name} and try #{@try.name}." end end end diff --git a/app/controllers/proxy_controller.rb b/app/controllers/proxy_controller.rb index 2afc84c51..90ecd9b85 100644 --- a/app/controllers/proxy_controller.rb +++ b/app/controllers/proxy_controller.rb @@ -115,6 +115,17 @@ def proxy_url_params params.require(:url) end + private + + def rack_header? header_name + predefined_rack_headers = %w[ + HTTP_VERSION HTTP_ACCEPT HTTP_ACCEPT_CHARSET HTTP_ACCEPT_ENCODING + HTTP_ACCEPT_LANGUAGE HTTP_CACHE_CONTROL HTTP_CONNECTION HTTP_HOST + HTTP_REFERER HTTP_USER_AGENT HTTP_X_REQUEST_ID + ] + + predefined_rack_headers.include?(header_name) + end end # rubocop:enable Layout/LineLength # rubocop:enable Metrics/AbcSize diff --git a/app/jobs/query_runner_job.rb b/app/jobs/query_runner_job.rb index 9d49eed01..5a84fc8ea 100644 --- a/app/jobs/query_runner_job.rb +++ b/app/jobs/query_runner_job.rb @@ -6,11 +6,11 @@ class QueryRunnerJob < ApplicationJob queue_as :bulk_processing - - def perform acase, try + # rubocop:disable Metrics/MethodLength + def perform acase, _try query_count = acase.queries.count - + acase.queries.each_with_index do |query, counter| Turbo::StreamsChannel.broadcast_render_to( :notifications, @@ -18,23 +18,23 @@ def perform acase, try partial: 'admin/query_runner/notification', locals: { query: query, query_count: query_count, counter: counter } ) - - sleep(2) end - + Turbo::StreamsChannel.broadcast_render_to( :notifications, target: 'notifications', partial: 'admin/query_runner/notification', locals: { query: nil, query_count: query_count, counter: 0 } ) - - end - - + # rubocop:enable Metrics/MethodLength + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/PerceivedComplexity def fetch excluded_keys = [ :url, :action, :controller, :proxy_debug ] @@ -103,7 +103,8 @@ def fetch response = connection.post do |req| req.path = uri.path query_params = request.query_parameters.except(*excluded_keys) - body_params = request.request_parameters.except(*query_params.keys) # not sure about this and the request.raw_post + # not sure about this and the request.raw_post + body_params = request.request_parameters.except(*query_params.keys) query_params.each do |param| req.params[param.first] = param.second end @@ -117,14 +118,17 @@ def fetch begin data = JSON.parse(response.body) - return {json: data, status: response.status} + { json: data, status: response.status } rescue JSON::ParserError # sometimes the API is returning plain old text, like a "Unauthorized" type message. - return {json: {response: response.body}, status: response.status} + { json: { response: response.body }, status: response.status } end rescue Faraday::ConnectionFailed => e - return {json: { proxy_error: e.message }, status: :internal_server_error} + { json: { proxy_error: e.message }, status: :internal_server_error } end end - + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/PerceivedComplexity end diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index 29953a292..4a9e7f7c8 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -5,165 +5,164 @@ class FetchService attr_reader :logger, :options - def initialize url,proxy_debug, opts = {} + def initialize opts = {} default_options = { - logger: Rails.logger + logger: Rails.logger, + snapshot_limit: 10, + debug_mode: false, } - @options = default_options.merge(opts.deep_symbolize_keys) + @options = default_options.merge(opts) - @url = url - @proxy_debug = proxy_debug @logger = @options[:logger] end - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/PerceivedComplexity - def validate - params_to_use = @data_to_process - scorer_name = params_to_use[:scorer][:name] - scorer = Scorer.find_by(name: scorer_name) - if scorer.nil? - @book.errors.add(:scorer, "with name '#{scorer_name}' needs to be migrated over first.") - else - @book.scorer = scorer - end + def begin acase, atry + @case = acase + @try = atry - selection_strategy_name = params_to_use[:selection_strategy][:name] - selection_strategy = SelectionStrategy.find_by(name: selection_strategy_name) - if selection_strategy.nil? - @book.errors.add(:selection_strategy, - "Selection strategy with name '#{selection_strategy_name}' needs to be migrated over first.") - else - @book.selection_strategy = selection_strategy - end - - if params_to_use[:query_doc_pairs] - list_of_emails_of_users = [] - params_to_use[:query_doc_pairs].each do |query_doc_pair| - next unless query_doc_pair[:judgements] - - query_doc_pair[:judgements].each do |judgement| - list_of_emails_of_users << judgement[:user_email] if judgement[:user_email].present? - end - end - list_of_emails_of_users.uniq! - list_of_emails_of_users.each do |email| - unless User.exists?(email: email) - if options[:force_create_users] - User.invite!({ email: email, password: '', skip_invitation: true }, @current_user) - else - @book.errors.add(:base, "User with email '#{email}' needs to be migrated over first.") - end - end - end - end + @snapshot = @case.snapshots.build(name: 'Fetch [BEGUN]') + @snapshot.scorer = @case.scorer + @snapshot.try = @try + @snapshot.save! + @snapshot end - def fetch - excluded_keys = [ :url, :action, :controller, :proxy_debug ] - - url_param = @url - - proxy_debug = 'true' == @proxy_debug + def store_query_results query, docs + snapshot_query = @snapshot.snapshot_queries.create(query: query, number_of_results: results.count) - uri = URI.parse(url_param) - url_without_path = "#{uri.scheme}://#{uri.host}" - url_without_path += ":#{uri.port}" unless uri.port.nil? + snapshot_manager = SnapshotManager.new(@snapshot) + snapshot_manager.setup_docs_for_query(snapshot_query, docs) + snapshot_query + end - connection = Faraday.new(url: url_without_path) do |faraday| - # Configure the connection options, such as headers or middleware - faraday.response :follow_redirects - faraday.response :logger, nil, { headers: proxy_debug, bodies: proxy_debug, errors: !Rails.env.test? } - faraday.ssl.verify = false - faraday.request :url_encoded + def complete + @snapshot.name = 'Fetch [COMPLETED]' + @snapshot.save! - matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } + # Keep the first snapshot, and the most recents, deleting the ones out of the middle. + # Not the best sampling! + snapshot_to_delete = @case.snapshots[1..((@options[:snapshot_limit] * -1) + @case.snapshots.count)] + snapshot_to_delete&.each(&:destroy) + end - matching_headers.each do |name, value| - converted_name = name.sub('HTTP_', '') - converted_name = converted_name.tr('_', '-') - faraday.headers[converted_name] = value - end + # rubocop:disable Metrics/MethodLength + def get_connection url, debug_mode, credentials, custom_headers + if @connection.nil? + @connection = Faraday.new(url: url) do |faraday| + # Configure the connection options, such as headers or middleware + faraday.response :follow_redirects + faraday.response :logger, nil, { headers: debug_mode, bodies: debug_mode, errors: !Rails.env.test? } + faraday.ssl.verify = false + # faraday.request :url_encoded + + # matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } + + # matching_headers.each do |name, value| + # converted_name = name.sub('HTTP_', '') + # converted_name = converted_name.tr('_', '-') + # faraday.headers[converted_name] = value + # end + + faraday.headers['Content-Type'] = 'application/json' + unless credentials.nil? + username, password = credentials.split(':') + faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" + end - faraday.headers['Content-Type'] = 'application/json' - has_credentials = !uri.userinfo.nil? - if has_credentials - username, password = uri.userinfo.split(':') - faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" + unless custom_headers.nil? + puts JSON.parse(custom_headers).to_h + JSON.parse(custom_headers).to_h.each do |key, value| + faraday.headers[key] = value + end + end + faraday.adapter Faraday.default_adapter end - faraday.adapter Faraday.default_adapter end + @connection + end + # rubocop:enable Metrics/MethodLength - begin - if request.get? - response = connection.get do |req| - req.path = uri.path - query_params = request.query_parameters.except(*excluded_keys) - body_params = request.request_parameters.except(*query_params.keys) - - query_params.each do |param| - req.params[param.first] = param.second - end - - # the url parameter often has a format like - # http://myserver.com/search?query=text, and when this is passed in - # we get http://localhost:3000/proxy/fetch?url=http://myserver.com/search?query=text&rows=10 - # which means the parameter "query=text" is lost because the URL is parsed and this part is dropped, - # so here we add this one parameter back in if we have it. - if url_param.include?('?') - # sometimes our url looks like http://myserver.com/search?q=tiger - # But it could also be http://myserver.com/search?q=tiger? and that needs handling via the special .split - extra_query_param = url_param.split('?', 2).last.split('=') - - req.params[extra_query_param.first] = extra_query_param.second - end - unless body_params.empty? - - json_query = body_params.first.first - req.body = json_query + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/PerceivedComplexity + def create_request atry, query + search_endpoint = atry.search_endpoint + debug_mode = true + connection = get_connection search_endpoint.endpoint_url, debug_mode, search_endpoint.basic_auth_credential, + search_endpoint.custom_headers + + http_verb = search_endpoint.api_method # .to_sym + + # TODO: Make api_method a enum + http_verb = :get if 'JSONP' == http_verb + http_verb = :get if 'GET' == http_verb + http_verb = :post if 'POST' == http_verb + http_verb = :put if 'PUT' == http_verb + puts "Running query with #{http_verb}" + + args = atry.args + + response = nil + # response = connection.run_request(http_verb, search_endpoint.endpoint_url) do |req| + case http_verb + when :get + response = connection.get do |req| + req.url search_endpoint.endpoint_url + args.each_value do |value| + value.map! { |val| val.gsub("\#$query##", query.query_text) } + + puts "Here is the key: #{key}" + puts "Here is the value: #{value}" + # if value.is_a?(Array) + # very unsure how we are handling the value, it may always be a array? + value.each do |item| + req.params[key] = item end end - elsif request.post? - response = connection.post do |req| - req.path = uri.path - query_params = request.query_parameters.except(*excluded_keys) - body_params = request.request_parameters.except(*query_params.keys) # not sure about this and the request.raw_post - query_params.each do |param| - req.params[param.first] = param.second - end - unless body_params.empty? - json_query = request.raw_post - req.body = json_query - end - end + # Add any additional headers, params, or other options as needed end - - begin - data = JSON.parse(response.body) - render json: data, status: response.status - rescue JSON::ParserError - # sometimes the API is returning plain old text, like a "Unauthorized" type message. - render plain: response.body, status: response.status + when :post + response = connection.post do |req| + req.url search_endpoint.endpoint_url + args = replace_values(args, query.query_text) + req.body = args + # Add any additional headers, params, or other options as needed + end + when :put + response = connection.put do |req| + req.url search_endpoint.endpoint_url + args = replace_values(args, query.query_text) + req.body = args + # Add any additional headers, params, or other options as needed end - rescue Faraday::ConnectionFailed => e - render json: { proxy_error: e.message }, status: :internal_server_error + else + raise "Invalid HTTP verb: #{http_verb}" end + + response end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/PerceivedComplexity - - private - - def rack_header? header_name - predefined_rack_headers = %w[ - HTTP_VERSION HTTP_ACCEPT HTTP_ACCEPT_CHARSET HTTP_ACCEPT_ENCODING - HTTP_ACCEPT_LANGUAGE HTTP_CACHE_CONTROL HTTP_CONNECTION HTTP_HOST - HTTP_REFERER HTTP_USER_AGENT HTTP_X_REQUEST_ID - ] - - predefined_rack_headers.include?(header_name) + # rubocop:disable Metrics/PerceivedComplexity + def replace_values data, query_text + if data.is_a?(Hash) + data.each do |key, value| + if "\#$query##" == value + data[key] = query_text + elsif value.is_a?(Hash) || value.is_a?(Array) + replace_values(value, query_text) + end + end + elsif data.is_a?(Array) + data.each { |item| replace_values(item, query_text) } + end + data end + # rubocop:enable Metrics/PerceivedComplexity end diff --git a/app/services/snapshot_manager.rb b/app/services/snapshot_manager.rb index 5d7f24229..c62b2544e 100644 --- a/app/services/snapshot_manager.rb +++ b/app/services/snapshot_manager.rb @@ -165,8 +165,6 @@ def csv_to_queries_hash docs query_docs end - private - # rubocop:disable Metrics/MethodLength def setup_docs_for_query query, docs results = [] @@ -193,6 +191,8 @@ def setup_docs_for_query query, docs end # rubocop:enable Metrics/MethodLength + private + def extract_doc_info row case @options[:format] when :csv diff --git a/config/routes.rb b/config/routes.rb index 0f9fc8ef0..d105e1dae 100755 --- a/config/routes.rb +++ b/config/routes.rb @@ -118,7 +118,7 @@ end resources :query_runner, only: [ :index ] do post 'run_queries', on: :collection - end + end end # preview routes for mailers diff --git a/test/fixtures/search_endpoints.yml b/test/fixtures/search_endpoints.yml index e090f4c6d..d613b85ca 100644 --- a/test/fixtures/search_endpoints.yml +++ b/test/fixtures/search_endpoints.yml @@ -134,8 +134,6 @@ try_with_headers: api_method: JSONP custom_headers: '{"Authorization": "ApiKey TEF5QkFJUUJVUnRsNG1fekNCR3E6WmRYUFJRRVdTMHlBVWotWFdqQWxuUQ=="}' - - bootstrap_try_1: name: Solr https://test.com endpoint_url: https://test.com diff --git a/test/lib/solr_arg_parser_test.rb b/test/lib/solr_arg_parser_test.rb index 5a9ecb687..6bca66261 100644 --- a/test/lib/solr_arg_parser_test.rb +++ b/test/lib/solr_arg_parser_test.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +o # frozen_string_literal: true + require 'test_helper' class SolrArgParserTest < ActiveSupport::TestCase diff --git a/test/services/fetch_service_test.rb b/test/services/fetch_service_test.rb new file mode 100644 index 000000000..6458e36c0 --- /dev/null +++ b/test/services/fetch_service_test.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require 'test_helper' + +class FetchServiceTest < ActiveSupport::TestCase + # Need to make the url. + # Need to understand the hTTP verb, GET, POST, JSONP + # Need to send the url with each query, swapping in the template. + # Need to also send the body with the qOptions as appropirate. + # Need to understand if it worked or not. + # Need to store the results as a snapshot. + # - Need ot use the snapshot structure to manage the lifecycle of the runs. + # + + let(:try_with_headers) { tries(:try_with_headers) } + let(:case_with_ratings) { cases(:random_case) } + let(:search_endpoint) { search_endpoints(:try_with_headers) } + let(:options) do + { + debug_mode: true, + fake_call: false, + snapshot_limit: 3, + } + end + + describe 'Creating an appropriate search query from Quepid data' do + let(:acase) { cases(:queries_case) } + let(:atry) { tries(:for_case_queries_case) } + let(:es_try_with_curator_vars) { tries(:es_try_with_curator_vars) } + let(:try_with_headers) { tries(:try_with_headers) } + let(:first_query) { queries(:first_query) } + + it 'creates a GET request' do + fetch_service = FetchService.new options + response = fetch_service.create_request(atry, first_query) + assert_not_nil response + assert 200 == response.status + end + + it 'works with custom headers and JSONP' do + fetch_service = FetchService.new options + response = fetch_service.create_request(try_with_headers, first_query) + assert_not_nil response + assert 200 == response.status + end + + it 'creates a POST request' do + fetch_service = FetchService.new options + response = fetch_service.create_request(es_try_with_curator_vars, first_query) + assert_not_nil response + assert 200 == response.status + end + end + + describe 'Running a fetch cycle produces a snapshot' do + let(:acase) { cases(:queries_case) } + let(:atry) { tries(:for_case_queries_case) } + let(:first_query) { queries(:first_query) } + let(:second_query) { queries(:second_query) } + + it 'creates a snapshot when you begin' do + fetch_service = FetchService.new options + assert_difference 'acase.snapshots.count' do + snapshot = fetch_service.begin(acase, atry) + assert snapshot.name.starts_with?('Fetch [BEGUN]') + end + end + + it 'saves a snapshot query for each run' do + fetch_service = FetchService.new options + fetch_service.begin(acase, atry) + + docs = [ + { id: 'doc1', explain: '1' }, + { id: 'doc2', explain: '2' } + ] + + assert_difference 'first_query.snapshot_queries.count' do + snapshot_query = fetch_service.store_query_results first_query, docs + assert_equal docs.size, snapshot_query.snapshot_docs.size + end + end + + it 'limits how many snapshots you can have when completed' do + fetch_service = FetchService.new options + assert_difference 'acase.snapshots.count', 6 do + 6.times do + fetch_service.begin(acase, atry) + end + end + + assert acase.snapshots.count > options[:snapshot_limit] + + acase.snapshots.first + + snapshot = fetch_service.begin(acase, atry) + + fetch_service.complete + + assert snapshot.name.starts_with?('Fetch [COMPLETED]') + assert_equal options[:snapshot_limit], acase.snapshots.count + end + end + + test 'think should what we have' do + FetchService.new options + end + + test 'should be able to handle a get' do + fetch_service = FetchService.new options + + # should url: be in params? Or seperate? + params = { + url: 'http://solr.quepid.com:8983/solr/statedecoded/select', fl: 'id,text', q: 'legal', rows: 10, start: 0 + } + + fetch_service.fetch params + assert_response :success + end + + test 'should track failure' do + end + + test 'should track successes' do + end +end diff --git a/test/support/webmock.rb b/test/support/webmock.rb index fe537364a..39a4f780a 100644 --- a/test/support/webmock.rb +++ b/test/support/webmock.rb @@ -208,6 +208,35 @@ def setup } ) .to_return(status: 200, body: '', headers: {}) + + # Testing out fetch service using + # search_endpoint for_case_queries_case + # try for_case_queries_case + stub_request(:get, 'http://test.com/solr/tmdb/select?q=First%20Query') + .with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/json', + 'User-Agent' => 'Faraday v2.9.0', + } + ) + .to_return(status: 200, body: mock_statedecoded_body, headers: {}) + + # Testing out fetch service using + # search_endpoint for_case_queries_case + # try es_try_with_curator_vars + stub_request(:post, 'http://test.com:9200/tmdb/_search') + .with( + body: { 'query'=>{ 'multi_match'=>{ 'fields' => 'title, overview', 'query' => 'First Query', 'tie_breaker' => '1' } } }, + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/json', + 'User-Agent' => 'Faraday v2.9.0', + } + ) + .to_return(status: 200, body: mock_statedecoded_body, headers: {}) end # rubocop:enable Metrics/MethodLength From 7d4e7d436721c6d17f9c5b1172ef5bdaf53d7a85 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Fri, 5 Jul 2024 14:35:59 +0200 Subject: [PATCH 007/120] rubocop --- app/services/fetch_service.rb | 2 ++ test/services/fetch_service_test.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index 4a9e7f7c8..42463300e 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# rubocop:disable Metrics/ClassLength class FetchService # include ProgressIndicator @@ -166,3 +167,4 @@ def replace_values data, query_text end # rubocop:enable Metrics/PerceivedComplexity end +# rubocop:enable Metrics/ClassLength diff --git a/test/services/fetch_service_test.rb b/test/services/fetch_service_test.rb index 6458e36c0..20a87903b 100644 --- a/test/services/fetch_service_test.rb +++ b/test/services/fetch_service_test.rb @@ -119,8 +119,10 @@ class FetchServiceTest < ActiveSupport::TestCase end test 'should track failure' do + assert true end test 'should track successes' do + assert true end end From 29a9a1468cfe86413e7df807ad356c7d294259f6 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 9 Jul 2024 18:57:44 +0200 Subject: [PATCH 008/120] Now storing the response status and body text. Practiced using the query runner job! --- app/jobs/query_runner_job.rb | 132 ++++----------- app/models/snapshot.rb | 8 +- app/models/snapshot_query.rb | 2 + app/services/fetch_service.rb | 159 ++++++++++++------ .../_notification_case.turbo_stream.erb | 17 ++ app/views/home/case_prophet.html.erb | 2 + ...add_response_fields_to_snapshot_queries.rb | 6 + db/schema.rb | 4 +- test/fixtures/queries.yml | 5 + test/fixtures/snapshot_queries.yml | 2 + test/lib/solr_arg_parser_test.rb | 2 - test/services/fetch_service_test.rb | 48 +++--- test/support/webmock.rb | 15 ++ 13 files changed, 212 insertions(+), 190 deletions(-) create mode 100644 app/views/admin/query_runner/_notification_case.turbo_stream.erb create mode 100644 db/migrate/20240709141551_add_response_fields_to_snapshot_queries.rb diff --git a/app/jobs/query_runner_job.rb b/app/jobs/query_runner_job.rb index 5a84fc8ea..fece081a1 100644 --- a/app/jobs/query_runner_job.rb +++ b/app/jobs/query_runner_job.rb @@ -8,10 +8,27 @@ class QueryRunnerJob < ApplicationJob queue_as :bulk_processing # rubocop:disable Metrics/MethodLength - def perform acase, _try + def perform acase, atry query_count = acase.queries.count + options = { + fake_mode: true, + debug_mode: true, + snapshot_limit: 3, + } + + fetch_service = FetchService.new options + fetch_service.begin(acase, atry) + acase.queries.each_with_index do |query, counter| + response = fetch_service.make_request(atry, query) + + response_code = response.status + response_body = response.body + + docs = [] + fetch_service.store_query_results query, docs, response_code, response_body + Turbo::StreamsChannel.broadcast_render_to( :notifications, target: 'notifications', @@ -19,6 +36,13 @@ def perform acase, _try locals: { query: query, query_count: query_count, counter: counter } ) + Turbo::StreamsChannel.broadcast_render_to( + :notifications, + target: "notifications-case-#{acase.id}", + partial: 'admin/query_runner/notification_case', + locals: { acase: acase, query: query, query_count: query_count, counter: counter } + ) + sleep(2) end @@ -28,107 +52,15 @@ def perform acase, _try partial: 'admin/query_runner/notification', locals: { query: nil, query_count: query_count, counter: 0 } ) - end - # rubocop:enable Metrics/MethodLength - - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/MethodLength - # rubocop:disable Metrics/PerceivedComplexity - def fetch - excluded_keys = [ :url, :action, :controller, :proxy_debug ] - - url_param = proxy_url_params - - proxy_debug = 'true' == params[:proxy_debug] - uri = URI.parse(url_param) - url_without_path = "#{uri.scheme}://#{uri.host}" - url_without_path += ":#{uri.port}" unless uri.port.nil? - - connection = Faraday.new(url: url_without_path) do |faraday| - # Configure the connection options, such as headers or middleware - faraday.response :follow_redirects - faraday.response :logger, nil, { headers: proxy_debug, bodies: proxy_debug, errors: !Rails.env.test? } - faraday.ssl.verify = false - faraday.request :url_encoded - - matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } - - matching_headers.each do |name, value| - converted_name = name.sub('HTTP_', '') - converted_name = converted_name.tr('_', '-') - faraday.headers[converted_name] = value - end - - faraday.headers['Content-Type'] = 'application/json' - has_credentials = !uri.userinfo.nil? - if has_credentials - username, password = uri.userinfo.split(':') - faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" - end - faraday.adapter Faraday.default_adapter - end - - begin - if request.get? - response = connection.get do |req| - req.path = uri.path - query_params = request.query_parameters.except(*excluded_keys) - body_params = request.request_parameters.except(*query_params.keys) - - query_params.each do |param| - req.params[param.first] = param.second - end - - # the url parameter often has a format like - # http://myserver.com/search?query=text, and when this is passed in - # we get http://localhost:3000/proxy/fetch?url=http://myserver.com/search?query=text&rows=10 - # which means the parameter "query=text" is lost because the URL is parsed and this part is dropped, - # so here we add this one parameter back in if we have it. - if url_param.include?('?') - # sometimes our url looks like http://myserver.com/search?q=tiger - # But it could also be http://myserver.com/search?q=tiger? and that needs handling via the special .split - extra_query_param = url_param.split('?', 2).last.split('=') - - req.params[extra_query_param.first] = extra_query_param.second - end - unless body_params.empty? - - json_query = body_params.first.first - req.body = json_query - end - end - elsif request.post? - response = connection.post do |req| - req.path = uri.path - query_params = request.query_parameters.except(*excluded_keys) - # not sure about this and the request.raw_post - body_params = request.request_parameters.except(*query_params.keys) - query_params.each do |param| - req.params[param.first] = param.second - end - unless body_params.empty? - json_query = request.raw_post - - req.body = json_query - end - end - end + Turbo::StreamsChannel.broadcast_render_to( + :notifications, + target: "notifications-case-#{acase.id}", + partial: 'admin/query_runner/notification_case', + locals: { acase: acase, query: nil, query_count: query_count, counter: 0 } + ) - begin - data = JSON.parse(response.body) - { json: data, status: response.status } - rescue JSON::ParserError - # sometimes the API is returning plain old text, like a "Unauthorized" type message. - { json: { response: response.body }, status: response.status } - end - rescue Faraday::ConnectionFailed => e - { json: { proxy_error: e.message }, status: :internal_server_error } - end + fetch_service.complete end - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Metrics/MethodLength - # rubocop:enable Metrics/PerceivedComplexity end diff --git a/app/models/snapshot.rb b/app/models/snapshot.rb index f3fe4eb3f..db5a4b958 100644 --- a/app/models/snapshot.rb +++ b/app/models/snapshot.rb @@ -30,11 +30,9 @@ class Snapshot < ApplicationRecord belongs_to :scorer, optional: true # shouldn't be optional! # see the call back delete_associated_objects for special delete logic. - # rubocop:disable Rails/HasManyOrHasOneDependent - has_many :snapshot_queries - # rubocop:enable Rails/HasManyOrHasOneDependent - has_many :snapshot_docs, - through: :snapshot_queries + has_many :snapshot_queries, dependent: :delete_all + has_many :snapshot_docs, + through: :snapshot_queries has_one_attached :snapshot_file diff --git a/app/models/snapshot_query.rb b/app/models/snapshot_query.rb index d28d2e0bd..ff2d947bb 100644 --- a/app/models/snapshot_query.rb +++ b/app/models/snapshot_query.rb @@ -7,6 +7,8 @@ # id :integer not null, primary key # all_rated :boolean # number_of_results :integer +# response_body :text(65535) +# response_status :integer # score :float(24) # query_id :integer # snapshot_id :integer diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index 42463300e..b0d27f090 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'faraday' +require 'faraday/follow_redirects' + # rubocop:disable Metrics/ClassLength class FetchService # include ProgressIndicator @@ -29,8 +32,13 @@ def begin acase, atry @snapshot end - def store_query_results query, docs - snapshot_query = @snapshot.snapshot_queries.create(query: query, number_of_results: results.count) + def store_query_results query, docs, response_status, response_body + snapshot_query = @snapshot.snapshot_queries.create( + query: query, + number_of_results: docs.count, + response_status: response_status, + response_body: response_body + ) snapshot_manager = SnapshotManager.new(@snapshot) snapshot_manager.setup_docs_for_query(snapshot_query, docs) @@ -49,38 +57,27 @@ def complete # rubocop:disable Metrics/MethodLength def get_connection url, debug_mode, credentials, custom_headers - if @connection.nil? - @connection = Faraday.new(url: url) do |faraday| - # Configure the connection options, such as headers or middleware - faraday.response :follow_redirects - faraday.response :logger, nil, { headers: debug_mode, bodies: debug_mode, errors: !Rails.env.test? } - faraday.ssl.verify = false - # faraday.request :url_encoded - - # matching_headers = request.headers.select { |name, _| name.start_with?('HTTP') && !rack_header?(name) } - - # matching_headers.each do |name, value| - # converted_name = name.sub('HTTP_', '') - # converted_name = converted_name.tr('_', '-') - # faraday.headers[converted_name] = value - # end - - faraday.headers['Content-Type'] = 'application/json' - unless credentials.nil? - username, password = credentials.split(':') - faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" - end + connection = Faraday.new(url: url) do |faraday| + # Configure the connection options, such as headers or middleware + faraday.response :follow_redirects + faraday.response :logger, nil, { headers: debug_mode, bodies: debug_mode, errors: !Rails.env.test? } + faraday.ssl.verify = false + + faraday.headers['Content-Type'] = 'application/json' + unless credentials.nil? + username, password = credentials.split(':') + faraday.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{username}:#{password}")}" + end - unless custom_headers.nil? - puts JSON.parse(custom_headers).to_h - JSON.parse(custom_headers).to_h.each do |key, value| - faraday.headers[key] = value - end + unless custom_headers.nil? + puts JSON.parse(custom_headers).to_h + JSON.parse(custom_headers).to_h.each do |key, value| + faraday.headers[key] = value end - faraday.adapter Faraday.default_adapter end + faraday.adapter Faraday.default_adapter end - @connection + connection end # rubocop:enable Metrics/MethodLength @@ -88,15 +85,24 @@ def get_connection url, debug_mode, credentials, custom_headers # rubocop:disable Metrics/CyclomaticComplexity # rubocop:disable Metrics/MethodLength # rubocop:disable Metrics/PerceivedComplexity - def create_request atry, query + def make_request atry, query + if @options[:fake_mode] + # useful in development mode when we don't have access to a search engine' + body = mock_response + response = Faraday::Response.new(status: 200, body: body) + return response + end search_endpoint = atry.search_endpoint - debug_mode = true - connection = get_connection search_endpoint.endpoint_url, debug_mode, search_endpoint.basic_auth_credential, - search_endpoint.custom_headers + if @connection.nil? + @connection = get_connection search_endpoint.endpoint_url, + @options[:debug_mode], + search_endpoint.basic_auth_credential, + search_endpoint.custom_headers + end - http_verb = search_endpoint.api_method # .to_sym + http_verb = search_endpoint.api_method - # TODO: Make api_method a enum + # TODO: Make api_method an enum http_verb = :get if 'JSONP' == http_verb http_verb = :get if 'GET' == http_verb http_verb = :post if 'POST' == http_verb @@ -109,35 +115,26 @@ def create_request atry, query # response = connection.run_request(http_verb, search_endpoint.endpoint_url) do |req| case http_verb when :get - response = connection.get do |req| + response = @connection.get do |req| req.url search_endpoint.endpoint_url - args.each_value do |value| - value.map! { |val| val.gsub("\#$query##", query.query_text) } - - puts "Here is the key: #{key}" - puts "Here is the value: #{value}" - # if value.is_a?(Array) - # very unsure how we are handling the value, it may always be a array? - value.each do |item| - req.params[key] = item + args.each do |key, value| + value.each do |val| + val.gsub!('#$query##', query.query_text) + req.params[key] = val end end - - # Add any additional headers, params, or other options as needed end when :post - response = connection.post do |req| + response = @connection.post do |req| req.url search_endpoint.endpoint_url args = replace_values(args, query.query_text) - req.body = args - # Add any additional headers, params, or other options as needed + req.body = args.to_json end when :put - response = connection.put do |req| + response = @connection.put do |req| req.url search_endpoint.endpoint_url args = replace_values(args, query.query_text) - req.body = args - # Add any additional headers, params, or other options as needed + req.body = args.to_json end else raise "Invalid HTTP verb: #{http_verb}" @@ -154,7 +151,7 @@ def create_request atry, query def replace_values data, query_text if data.is_a?(Hash) data.each do |key, value| - if "\#$query##" == value + if '#$query##' == value data[key] = query_text elsif value.is_a?(Hash) || value.is_a?(Array) replace_values(value, query_text) @@ -166,5 +163,57 @@ def replace_values data, query_text data end # rubocop:enable Metrics/PerceivedComplexity + + # rubocop:disable Metrics/MethodLength + # rubocop:disable Layout/LineLength + def mock_response + mock_statedecoded_body = '{ + "responseHeader":{ + "zkConnected":true, + "status":0, + "QTime":0, + "params":{ + "q":"*:*", + "fl":"id,text", + "start":"0", + "rows":"10"}}, + "response":{"numFound":20148,"start":0,"docs":[ + { + "id":"l_11730", + "text":"For the purpose of this chapter:\n\n"Claimant" means the person filing a claim pursuant to this chapter.\n\n"Commission" means the Virginia Workers' Compensation Commission.\n\n"Crime" means an act committed by any person in the Commonwealth of Virginia which would constitute a crime as defined by the Code of Virginia or at common law. However, no act involving the operation of a motor vehicle which results in injury shall constitute a crime for the purpose of this chapter unless the injuries (i) were intentionally inflicted through the use of such vehicle or (ii) resulted from a violation of § 18.2-51.4 or 18.2-266 or from a felony violation of § 46.2-894.\n\n"Family," when used with reference to a person, means (i) any person related to such person within the third degree of consanguinity or affinity, (ii) any person residing in the same household with such person, or (iii) a spouse.\n\n"Sexual abuse" means sexual abuse as defined in subdivision 6 of § 18.2-67.10 and acts constituting rape, sodomy, object sexual penetration or sexual battery as defined in Article 7 (§ 18.2-61 et seq.) of Chapter 4 of Title 18.2.\n\n"Victim" means a person who suffers personal physical injury or death as a direct result of a crime including a person who is injured or killed as a result of foreign terrorism or who suffers personal emotional injury as a direct result of being the subject of a violent felony offense as defined in subsection C of § 17.1-805, or stalking as described in § 18.2-60.3, or attempted robbery or abduction."}, + { + "id":"l_5780", + "text":"Agencies authorized under any other law to issue grading, building, or other permits for activities involving land-disturbing activities regulated under this article may not issue any such permit unless the applicant submits with his application an approved erosion and sediment control plan and certification that the plan will be followed and, upon the development of an online reporting system by the Department but no later than July 1, 2014, evidence of Virginia stormwater management state permit coverage where it is required. Prior to issuance of any permit, the agency may also require an applicant to submit a reasonable performance bond with surety, cash escrow, letter of credit, any combination thereof, or such other legal arrangement acceptable to the agency, to ensure that measures could be taken by the agency at the applicant's expense should he fail, after proper notice, within the time specified to initiate or maintain appropriate conservation action which may be required of him by the approved plan as a result of his land-disturbing activity. The amount of the bond or other security for performance shall not exceed the total of the estimated cost to initiate and maintain appropriate conservation action based on unit price for new public or private sector construction in the locality and a reasonable allowance for estimated administrative costs and inflation which shall not exceed 25 percent of the estimated cost of the conservation action. If the agency takes such conservation action upon such failure by the permittee, the agency may collect from the permittee for the difference should the amount of the reasonable cost of such action exceed the amount of the security held. Within 60 days of the achievement of adequate stabilization of the land-disturbing activity in any project or section thereof, the bond, cash escrow, letter of credit or other legal arrangement, or the unexpended or unobligated portion thereof, shall be refunded to the applicant or terminated based upon the percentage of stabilization accomplished in the project or section thereof. These requirements are in addition to all other provisions of law relating to the issuance of such permits and are not intended to otherwise affect the requirements for such permits."}, + { + "id":"l_16271", + "text":"If in the administration of this article a question concerning compliance with standards of practice governing any health care profession arises pursuant to Subtitle III (§ 54.1-2400 et seq.) of Title 54.1, the Commissioner or his designee shall consult with the appropriate health regulatory board within the Department of Health Professions."}, + { + "id":"l_4010", + "text":"The production of documentary material in response to a civil investigative demand served under this article shall be made under a sworn certificate, in such form as the demand designates, by (i) in the case of a natural person, the person to whom the demand is directed, or (ii) in the case of a person other than a natural person, a person having knowledge of the facts and circumstances relating to such production and authorized to act on behalf of such person. The certificate shall state that all of the documentary material required by the demand and in the possession, custody, or control of the person to whom the demand is directed has been produced and made available to the investigator identified in the demand.\n\nAny person upon whom any civil investigative demand for the production of documentary material has been served shall make such material available for inspection and copying to the investigator identified in such demand at the principal place of business of such person, or at such other place as the investigator and the person thereafter may agree and prescribe in writing, or as the court may direct. Such material shall be made available on the return date specified in such demand, or on such later date as the investigator may prescribe in writing. Such person may, upon written agreement between the person and the investigator, substitute copies for originals of all or any part of such material."}, + { + "id":"l_5552", + "text":"For purposes of computing fire protection or law-enforcement employees' entitlement to overtime compensation, all hours that an employee works or is in a paid status during his regularly scheduled work hours shall be counted as hours of work. The provisions of this section pertaining to law-enforcement employees shall only apply to such employees of an employer of 100 or more law-enforcement employees."}, + { + "id":"l_725", + "text":"There is hereby created a political subdivision and public body corporate and politic of the Commonwealth of Virginia to be known as the Fort Monroe Authority, to be governed by a Board of Trustees (Board) consisting of 12 voting members appointed as follows: the Secretary of Natural Resources, the Secretary of Commerce and Trade, and the Secretary of Veterans Affairs and Homeland Security, or their successor positions if those positions no longer exist, from the Governor's cabinet; the member of the Senate of Virginia and the member of the House of Delegates representing the district in which Fort Monroe lies; two members appointed by the Hampton City Council; and five nonlegislative citizen members appointed by the Governor, four of whom shall have expertise relevant to the implementation of the Fort Monroe Reuse Plan, including but not limited to the fields of historic preservation, tourism, environment, real estate, finance, and education, and one of whom shall be a citizen representative from the Hampton Roads region. Cabinet members and elected representatives shall serve terms commensurate with their terms of office. Citizen appointees shall initially be appointed for staggered terms of either one, two, or three years, and thereafter shall serve for four-year terms. Cabinet members shall be entitled to send their deputies or another cabinet member, and legislative members another legislator, to meetings as full voting members in the event that official duties require their presence elsewhere.\n\nThe Board so appointed shall enter upon the performance of its duties and shall initially and annually thereafter elect one of its members as chairman and another as vice-chairman. The Board shall also elect annually a secretary, who shall be a member of the Board, and a treasurer, who need not be a member of the Board, or a secretary-treasurer, who need not be a member of the Board. The chairman, or in his absence the vice-chairman, shall preside at all meetings of the Board, and in the absence of both the chairman and vice-chairman, the Board shall elect a chairman pro tempore who shall preside at such meetings. Seven Trustees shall constitute a quorum, and all action by the Board shall require the affirmative vote of a majority of the Trustees present and voting, except that any action to amend or terminate the existing Reuse Plan, or to adopt a new Reuse Plan, shall require the affirmative vote of 75 percent or more of the Trustees present and voting. The members of the Board shall be entitled to reimbursement for expenses incurred in attendance upon meetings of the Board or while otherwise engaged in the discharge of their duties. Such expenses shall be paid out of the treasury of the Authority in such manner as shall be prescribed by the Authority."}, + { + "id":"l_2980", + "text":"No provision of this chapter imposing any liability shall apply to any act done or omitted in good faith in conformity with any rule, regulation, or interpretation thereof by the Commission or by the Federal Reserve Board or officer or employee duly authorized by the Board to issue such interpretation or approvals under the comparable provisions of the federal Equal Credit Opportunity Act, (15 U.S.C. § 1691 et seq.), and regulations thereunder, notwithstanding that after such act or omission has occurred, such rule, regulation, or interpretation is amended, rescinded, or determined by judicial or other authority to be invalid for any reason."}, + { + "id":"l_25249", + "text":"
In this section, "document of rescission" means a document stating that an identified satisfaction, certificate of satisfaction, or affidavit of satisfaction of a security instrument was recorded erroneously or fraudulently, the secured obligation remains unsatisfied, and the security instrument remains in force.
If a person records a satisfaction, certificate of satisfaction, or affidavit of satisfaction of a security instrument in error or by fraud, the person may execute and record a document of rescission. Upon recording, the document rescinds an erroneously recorded satisfaction, certificate, or affidavit.
A recorded document of rescission has no effect on the rights of a person who:
Acquired an interest in the real property described in a security instrument after the recording of the satisfaction, certificate of satisfaction, or affidavit of satisfaction of the security instrument and before the recording of the document of rescission; and
Would otherwise have priority over or take free of the lien created by the security instrument under the laws of the Commonwealth of Virginia.
A person, other than the clerk of the circuit court or any of his employees or other governmental official in the course of the performance of his recordation duties, who erroneously, fraudulently, or wrongfully records a document of rescission is subject to liability under § 55-66.3.
"}, + { + "id":"l_1063", + "text":"
All state public bodies created in the executive branch of state government and subject to the provisions of this chapter shall make available the following information to the public upon request and shall post such information on the Internet:
A plain English explanation of the rights of a requester under this chapter, the procedures to obtain public records from the public body, and the responsibilities of the public body in complying with this chapter. For purposes of this subdivision "plain English" means written in nontechnical, readily understandable language using words of common everyday usage and avoiding legal terms and phrases or other terms and words of art whose usage or special meaning primarily is limited to a particular field or profession;
Contact information for the person designated by the public body to (i) assist a requester in making a request for records or (ii) respond to requests for public records;
A general description, summary, list, or index of the types of public records maintained by such state public body;
A general description, summary, list, or index of any exemptions in law that permit or require such public records to be withheld from release; and
Any policy the public body has concerning the type of public records it routinely withholds from release as permitted by this chapter or other law.
The Freedom of Information Advisory Council, created pursuant to § 30-178, shall assist in the development and implementation of the provisions of subsection A, upon request.
"}, + { + "id":"l_20837", + "text":"
The Inspector shall administer the laws and regulations and shall have access to all records and properties necessary for this purpose. He shall perform all duties delegated by the Director pursuant to § 45.1-161.5 and maintain permanent records of the following:
Each application for a gas, oil, or geophysical operation and each permitted gas, oil, or geophysical operation;
Meetings, actions and orders of the Board;
Petitions for mining coal within 200 feet of or through a well;
Requests for special plugging by a coal owner or coal operator; and
All other records prepared pursuant to this chapter.
The Inspector shall serve as the principal executive of the staff of the Board.
The Inspector may take charge of well or corehole, or pipeline emergency operations whenever a well or corehole blowout, release of hydrogen sulfide or other gases, or other serious accident occurs.
"}] + }} + ' + + mock_statedecoded_body + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Layout/LineLength end # rubocop:enable Metrics/ClassLength diff --git a/app/views/admin/query_runner/_notification_case.turbo_stream.erb b/app/views/admin/query_runner/_notification_case.turbo_stream.erb new file mode 100644 index 000000000..184ddc6a9 --- /dev/null +++ b/app/views/admin/query_runner/_notification_case.turbo_stream.erb @@ -0,0 +1,17 @@ +<%= turbo_stream.replace "notification-case-#{acase.id}" do %> + + <% unless query.nil? %> + Query: <%=query.query_text%> + <% end %> + <% if counter > 0 %> + + <% else %> + + <% end %> + +<% end %> diff --git a/app/views/home/case_prophet.html.erb b/app/views/home/case_prophet.html.erb index 07e6ce8f2..83246deba 100644 --- a/app/views/home/case_prophet.html.erb +++ b/app/views/home/case_prophet.html.erb @@ -1,6 +1,8 @@
<%= number_with_precision(@case.last_score.score, precision: 2) unless @case.scores.empty? %> <%= @case.scorer.name %>

+ + <% if @case.scores.empty? %> no scores yet <% else %> diff --git a/db/migrate/20240709141551_add_response_fields_to_snapshot_queries.rb b/db/migrate/20240709141551_add_response_fields_to_snapshot_queries.rb new file mode 100644 index 000000000..ee1230529 --- /dev/null +++ b/db/migrate/20240709141551_add_response_fields_to_snapshot_queries.rb @@ -0,0 +1,6 @@ +class AddResponseFieldsToSnapshotQueries < ActiveRecord::Migration[7.1] + def change + add_column :snapshot_queries, :response_status, :integer + add_column :snapshot_queries, :response_body, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 116dc63f7..7f70c1063 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_06_26_181338) do +ActiveRecord::Schema[7.1].define(version: 2024_07_09_141551) do create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_bin", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false @@ -256,6 +256,8 @@ t.float "score" t.boolean "all_rated" t.integer "number_of_results" + t.integer "response_status" + t.text "response_body" t.index ["query_id"], name: "query_id" t.index ["snapshot_id"], name: "snapshot_id" end diff --git a/test/fixtures/queries.yml b/test/fixtures/queries.yml index 9450a670f..b232733e9 100644 --- a/test/fixtures/queries.yml +++ b/test/fixtures/queries.yml @@ -51,6 +51,11 @@ fourth_query: case: :queries_case query_text: Fourth Query arranged_at: 4 + +blowup_query: + case: :queries_case + query_text: BLOWUP_QUERY + arranged_at: 5 a_query: case: :snapshot_case diff --git a/test/fixtures/snapshot_queries.yml b/test/fixtures/snapshot_queries.yml index 5a6f36c4c..f72f23f34 100644 --- a/test/fixtures/snapshot_queries.yml +++ b/test/fixtures/snapshot_queries.yml @@ -5,6 +5,8 @@ # id :integer not null, primary key # all_rated :boolean # number_of_results :integer +# response_body :text(65535) +# response_status :integer # score :float(24) # query_id :integer # snapshot_id :integer diff --git a/test/lib/solr_arg_parser_test.rb b/test/lib/solr_arg_parser_test.rb index 6bca66261..5a9ecb687 100644 --- a/test/lib/solr_arg_parser_test.rb +++ b/test/lib/solr_arg_parser_test.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -o # frozen_string_literal: true - require 'test_helper' class SolrArgParserTest < ActiveSupport::TestCase diff --git a/test/services/fetch_service_test.rb b/test/services/fetch_service_test.rb index 20a87903b..f75d7b140 100644 --- a/test/services/fetch_service_test.rb +++ b/test/services/fetch_service_test.rb @@ -18,7 +18,6 @@ class FetchServiceTest < ActiveSupport::TestCase let(:options) do { debug_mode: true, - fake_call: false, snapshot_limit: 3, } end @@ -29,27 +28,35 @@ class FetchServiceTest < ActiveSupport::TestCase let(:es_try_with_curator_vars) { tries(:es_try_with_curator_vars) } let(:try_with_headers) { tries(:try_with_headers) } let(:first_query) { queries(:first_query) } + let(:blowup_query) { queries(:blowup_query) } it 'creates a GET request' do fetch_service = FetchService.new options - response = fetch_service.create_request(atry, first_query) + response = fetch_service.make_request(atry, first_query) assert_not_nil response assert 200 == response.status end it 'works with custom headers and JSONP' do fetch_service = FetchService.new options - response = fetch_service.create_request(try_with_headers, first_query) + response = fetch_service.make_request(try_with_headers, first_query) assert_not_nil response assert 200 == response.status end it 'creates a POST request' do fetch_service = FetchService.new options - response = fetch_service.create_request(es_try_with_curator_vars, first_query) + response = fetch_service.make_request(es_try_with_curator_vars, first_query) assert_not_nil response assert 200 == response.status end + + it 'handles a failed search request' do + fetch_service = FetchService.new options + response = fetch_service.make_request(atry, blowup_query) + assert_not_nil response + assert 404 == response.status + end end describe 'Running a fetch cycle produces a snapshot' do @@ -75,9 +82,13 @@ class FetchServiceTest < ActiveSupport::TestCase { id: 'doc2', explain: '2' } ] + response_status = 200 + response_body = '' + assert_difference 'first_query.snapshot_queries.count' do - snapshot_query = fetch_service.store_query_results first_query, docs + snapshot_query = fetch_service.store_query_results first_query, docs, response_status, response_body assert_equal docs.size, snapshot_query.snapshot_docs.size + assert_equal response_status, snapshot_query.response_status end end @@ -102,27 +113,10 @@ class FetchServiceTest < ActiveSupport::TestCase end end - test 'think should what we have' do - FetchService.new options - end - - test 'should be able to handle a get' do - fetch_service = FetchService.new options - - # should url: be in params? Or seperate? - params = { - url: 'http://solr.quepid.com:8983/solr/statedecoded/select', fl: 'id,text', q: 'legal', rows: 10, start: 0 - } - - fetch_service.fetch params - assert_response :success - end - - test 'should track failure' do - assert true - end - - test 'should track successes' do - assert true + describe 'parsing a response' do + it 'lets you create a resopnse' do + response = Faraday::Response.new(status: 200) + assert_equal 200, response.status + end end end diff --git a/test/support/webmock.rb b/test/support/webmock.rb index 39a4f780a..27682cfe0 100644 --- a/test/support/webmock.rb +++ b/test/support/webmock.rb @@ -237,6 +237,21 @@ def setup } ) .to_return(status: 200, body: mock_statedecoded_body, headers: {}) + + # Testing out fetch service using + # search_endpoint for_case_queries_case + # try es_try_with_curator_vars + # query blowup_query + stub_request(:get, 'http://test.com/solr/tmdb/select?q=BLOWUP_QUERY') + .with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Content-Type' => 'application/json', + 'User-Agent' => 'Faraday v2.9.0', + } + ) + .to_return(status: 404, body: '', headers: {}) end # rubocop:enable Metrics/MethodLength From a69b32b7ceee176c63a73728a1bd0bd3cc7d4d7b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 11 Jul 2024 16:30:51 +0200 Subject: [PATCH 009/120] Use new sidekiq, which better supports long jobs! --- Gemfile.lock | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ec9257f1f..8fb5d8948 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -127,7 +127,7 @@ GEM cmdstan (0.3.1) csv colorize (1.1.0) - concurrent-ruby (1.3.1) + concurrent-ruby (1.3.3) connection_pool (2.4.1) cookies_eu (1.7.8) js_cookie_rails (~> 2.2.0) @@ -226,6 +226,7 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) local_time (3.0.2) + logger (1.6.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -435,11 +436,12 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - sidekiq (7.2.4) + sidekiq (7.3.0) concurrent-ruby (< 2) connection_pool (>= 2.3.0) + logger rack (>= 2.2.4) - redis-client (>= 0.19.0) + redis-client (>= 0.22.2) sidekiq-limit_fetch (4.4.1) sidekiq (>= 6) simplecov (0.22.0) From 19e4f824665ad19f347047dce7149c5d96379004 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 11 Jul 2024 17:02:33 +0200 Subject: [PATCH 010/120] we had a reason for why this was the way it was! --- app/models/snapshot.rb | 4 +++- app/services/fetch_service.rb | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/snapshot.rb b/app/models/snapshot.rb index db5a4b958..29ae84e9c 100644 --- a/app/models/snapshot.rb +++ b/app/models/snapshot.rb @@ -30,7 +30,9 @@ class Snapshot < ApplicationRecord belongs_to :scorer, optional: true # shouldn't be optional! # see the call back delete_associated_objects for special delete logic. - has_many :snapshot_queries, dependent: :delete_all + # rubocop:disable Rails/HasManyOrHasOneDependent + has_many :snapshot_queries + # rubocop:enable Rails/HasManyOrHasOneDependent has_many :snapshot_docs, through: :snapshot_queries diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index b0d27f090..e61bf7e04 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -52,7 +52,8 @@ def complete # Keep the first snapshot, and the most recents, deleting the ones out of the middle. # Not the best sampling! snapshot_to_delete = @case.snapshots[1..((@options[:snapshot_limit] * -1) + @case.snapshots.count)] - snapshot_to_delete&.each(&:destroy) + # snapshot_to_delete&.each(&:destroy) + snapshot_to_delete.each(&:destroy!) end # rubocop:disable Metrics/MethodLength From cc49698088c013150bed84af20c21b2f1989b70c Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 10 Sep 2024 16:25:50 -0400 Subject: [PATCH 011/120] I hope this is good stuff --- app/models/snapshot.rb | 7 ++++--- app/models/snapshot_query.rb | 2 +- test/models/snapshot_test.rb | 24 ++++++++++++++++++++++++ test/services/fetch_service_test.rb | 8 ++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/app/models/snapshot.rb b/app/models/snapshot.rb index 29ae84e9c..4743c7a77 100644 --- a/app/models/snapshot.rb +++ b/app/models/snapshot.rb @@ -31,10 +31,10 @@ class Snapshot < ApplicationRecord # see the call back delete_associated_objects for special delete logic. # rubocop:disable Rails/HasManyOrHasOneDependent - has_many :snapshot_queries + has_many :snapshot_queries, dependent: :destroy # rubocop:enable Rails/HasManyOrHasOneDependent has_many :snapshot_docs, - through: :snapshot_queries + through: :snapshot_queries, dependent: :destroy has_one_attached :snapshot_file @@ -43,7 +43,7 @@ class Snapshot < ApplicationRecord # Callbacks before_validation :set_defaults - before_destroy :delete_associated_objects + #before_destroy :delete_associated_objects private @@ -52,6 +52,7 @@ def set_defaults end def delete_associated_objects + puts "I am about to delete associated objects" SnapshotDoc.joins(snapshot_query: :snapshot) .where(snapshot_queries: { snapshot: self }) .delete_all diff --git a/app/models/snapshot_query.rb b/app/models/snapshot_query.rb index ff2d947bb..d58ff645e 100644 --- a/app/models/snapshot_query.rb +++ b/app/models/snapshot_query.rb @@ -28,6 +28,6 @@ class SnapshotQuery < ApplicationRecord belongs_to :snapshot, optional: true # shouldn't be belongs_to :query, optional: true # shouldn't be has_many :snapshot_docs, -> { order(position: :asc) }, - dependent: :delete_all, + dependent: :destroy, inverse_of: :snapshot_query end diff --git a/test/models/snapshot_test.rb b/test/models/snapshot_test.rb index 0cc5405d9..d3ea53d90 100644 --- a/test/models/snapshot_test.rb +++ b/test/models/snapshot_test.rb @@ -48,4 +48,28 @@ class SnapshotTest < ActiveSupport::TestCase assert_equal name, snapshot.name end end + + describe 'deleting a snapshot' do + let(:asnapshot) { snapshots(:a_snapshot) } + let(:snapshot_query) { snapshot_queries(:first_snapshot_query) } + + test 'deleting a snapshot ALSO deletes the queries' do + require 'pp' + + pp snapshot_query + + pp asnapshot + + assert asnapshot.destroy! + + assert_raises(ActiveRecord::RecordNotFound) do + SnapshotQuery.find(snapshot_query.id) + end + + + #snapshot = Snapshot.create(case: acase) + + #assert_equal "Snapshot #{Time.zone.now.strftime('%D')}", snapshot.name + end + end end diff --git a/test/services/fetch_service_test.rb b/test/services/fetch_service_test.rb index f75d7b140..842cf0986 100644 --- a/test/services/fetch_service_test.rb +++ b/test/services/fetch_service_test.rb @@ -106,10 +106,18 @@ class FetchServiceTest < ActiveSupport::TestCase snapshot = fetch_service.begin(acase, atry) + count_of_snapshot_queries = SnapshotQuery.all.count + fetch_service.complete assert snapshot.name.starts_with?('Fetch [COMPLETED]') assert_equal options[:snapshot_limit], acase.snapshots.count + + count_of_snapshot_queries_after = SnapshotQuery.all.count + + puts "We had #{count_of_snapshot_queries} and now have #{count_of_snapshot_queries_after}" + + assert count_of_snapshot_queries > count_of_snapshot_queries_after end end From 5475178ff960391985beb7c3bcba777b113bf75e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:05:03 -0500 Subject: [PATCH 012/120] Nicer messaging of what to do next.. --- lib/tasks/sample_data.thor | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tasks/sample_data.thor b/lib/tasks/sample_data.thor index d5b4994d3..4c3eb9819 100644 --- a/lib/tasks/sample_data.thor +++ b/lib/tasks/sample_data.thor @@ -308,6 +308,8 @@ class SampleData < Thor end print_step 'End of multiple cases................' + print_step '' + print_step 'Now open http://localhost:3000 and login with quepid+realisticactivity@o19s.com / password' end desc 'large_data', 'load the very large sample data' From 01564c718559cda1269ff1f261895033175bd458 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:05:36 -0500 Subject: [PATCH 013/120] Add to the example some docs on this property --- .env.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 8484171f7..bbd37bc8a 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,10 @@ SOLID_CABLE_POLLING=0.1.seconds # /case page. FORCE_SSL=false +# This makes Quepid show detailed error messages in the UI instead of a generic 500 page, +# useful while testing a deployment in Production. +QUEPID_CONSIDER_ALL_REQUESTS_LOCAL=false + DB_HOST=mysql DB_USERNAME=root DB_PASSWORD=password From 4b37544a965d318911fbb7af400d846bdf8a8856 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:06:05 -0500 Subject: [PATCH 014/120] Finish work to show all the jobs that have a status --- app/views/books/show.html.erb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/views/books/show.html.erb b/app/views/books/show.html.erb index f014123e8..4358c0cc3 100644 --- a/app/views/books/show.html.erb +++ b/app/views/books/show.html.erb @@ -68,14 +68,20 @@ This book consists of <%= @book.query_doc_pairs.count %> query document pairs an Associated Files: <% if @book.populate_file.attached? %> <%= link_to 'Populate file', rails_blob_path(@book.populate_file.blob, only_path: true) %> made <%= time_ago_in_words @book.populate_file.created_at %> ago. + <%if @book.populate_job %> + There is currently a populate job for this book set up. It is <%= @book.populate_job %>. + <% end %> <% end %> <% if @book.import_file.attached? %> <%= link_to 'Import file', rails_blob_path(@book.import_file.blob, only_path: true) %> made <%= time_ago_in_words @book.import_file.created_at %> ago. <%if @book.import_job %> - There is currently a import job for this book. It is <%=@book.import_job%>. + There is currently a import job for this book set up. It is <%= @book.import_job %>. <% end %> <% end %> <% if @book.export_file.attached? %> + <%if @book.export_job %> + There is currently a export job for this book set up. It is <%= @book.export_job %>. + <% end %> <%= link_to 'Export file', rails_blob_path(@book.export_file.blob, only_path: true) %> made <%= time_ago_in_words @book.export_file.created_at %> ago. <% end %>

From 06920e331bb37e8ffba70f39860d8b32d2be6742 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:06:12 -0500 Subject: [PATCH 015/120] Fix image path --- app/views/sessions/new.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/sessions/new.html.erb b/app/views/sessions/new.html.erb index 20412f9a4..2d12b9956 100644 --- a/app/views/sessions/new.html.erb +++ b/app/views/sessions/new.html.erb @@ -21,7 +21,7 @@

<%= button_to user_keycloakopenid_omniauth_authorize_path, data: { turbo: "false" }, :class=> "btn btn-social btn-openid" do %> - <%= image_tag "openid-logo", :class => "black-to-white" %> Sign in with OpenId + <%= image_tag "openid-logo.png", :class => "black-to-white" %> Sign in with OpenId <% end %>

From c2ac54991a91d4925c5357c2f199dec978afe90a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:07:01 -0500 Subject: [PATCH 016/120] Fix set up for production --- config/environments/production.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 3b1ec3dc1..a91a8988f 100755 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -75,9 +75,9 @@ # Replace the default in-process memory cache store with a durable alternative. # config.cache_store = :mem_cache_store - # Replace the default in-process and non-durable queuing backend for Active Job. config.active_job.queue_adapter = :solid_queue - config.solid_queue.connects_to = { database: { writing: :primary } } + config.solid_queue.connects_to = { database: { writing: :primary, reading: :primary } } + config.solid_queue.silence_polling = true # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. From ec4e4fa54aab139097ab48af3d555730b6501ac9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 12:07:26 -0500 Subject: [PATCH 017/120] Bump solid_cable to work better in dev mode --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 43be6f72c..94c042614 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -142,7 +142,7 @@ GEM csv (3.3.0) d3-rails (3.5.17) railties (>= 3.1) - date (3.4.0) + date (3.4.1) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) @@ -470,7 +470,7 @@ GEM snaky_hash (2.0.1) hashie version_gem (~> 1.1, >= 1.1.1) - solid_cable (3.0.3) + solid_cable (3.0.4) actioncable (>= 7.2) activejob (>= 7.2) activerecord (>= 7.2) From 50c514f42ca62c9db4fbd0101da63d1203749d3b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sat, 7 Dec 2024 15:26:41 -0500 Subject: [PATCH 018/120] Lots of changes --- app/helpers/application_helper.rb | 3 ++- app/jobs/populate_book_job.rb | 11 ++++++++-- app/jobs/query_runner_job.rb | 6 +++--- app/models/snapshot.rb | 10 ++++----- app/services/book_importer.rb | 21 ++++++++++++------- app/services/fetch_service.rb | 7 ++++--- .../_notification.turbo_stream.erb | 17 ++++++++------- .../_notification_case.turbo_stream.erb | 12 +++++------ app/views/admin/query_runner/index.html.erb | 3 ++- app/views/books/_blah.turbo_stream.erb | 13 +++++++----- app/views/home/case_prophet.html.erb | 5 ++--- test/models/snapshot_test.rb | 17 +++++++-------- test/services/fetch_service_test.rb | 14 ++++++------- 13 files changed, 77 insertions(+), 62 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 65bd65045..03c24264c 100755 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -88,7 +88,8 @@ def form_for record, options = {}, &block end # rubocop:enable Naming/BlockForwarding - def form_with(model: nil, **options, &block) + # not sure this as "form_with" was helping, but we'll see.. + def form_with_disabled(model: nil, **options, &block) if options[:html].nil? options[:html] = { data: { turbo: false } } elsif options[:html][:data].nil? diff --git a/app/jobs/populate_book_job.rb b/app/jobs/populate_book_job.rb index 792d9847d..0a1952ead 100644 --- a/app/jobs/populate_book_job.rb +++ b/app/jobs/populate_book_job.rb @@ -17,7 +17,9 @@ def perform user, book, kase is_book_empty = book.query_doc_pairs.empty? - counter = params[:query_doc_pairs].size + total = params[:query_doc_pairs].size + counter = total + last_percent = 0 params[:query_doc_pairs].each do |pair| counter -= 1 query_doc_pair = book.query_doc_pairs.find_or_create_by query_text: pair[:query_text], @@ -43,11 +45,16 @@ def perform user, book, kase query_doc_pair.save! + # emit a message every percent that we cross, from 0 to 100... + percent = (((total - counter).to_f / total) * 100).truncate + next unless percent > last_percent + + last_percent = percent Turbo::StreamsChannel.broadcast_render_to( :notifications, target: 'notifications', partial: 'books/blah', - locals: { book: book, counter: counter, qdp: query_doc_pair } + locals: { book: book, counter: counter, percent: percent, qdp: query_doc_pair } ) end book.populate_file.purge diff --git a/app/jobs/query_runner_job.rb b/app/jobs/query_runner_job.rb index fece081a1..72dcad88f 100644 --- a/app/jobs/query_runner_job.rb +++ b/app/jobs/query_runner_job.rb @@ -43,21 +43,21 @@ def perform acase, atry locals: { acase: acase, query: query, query_count: query_count, counter: counter } ) - sleep(2) + sleep(0.1) end Turbo::StreamsChannel.broadcast_render_to( :notifications, target: 'notifications', partial: 'admin/query_runner/notification', - locals: { query: nil, query_count: query_count, counter: 0 } + locals: { query: nil, query_count: query_count, counter: -1 } ) Turbo::StreamsChannel.broadcast_render_to( :notifications, target: "notifications-case-#{acase.id}", partial: 'admin/query_runner/notification_case', - locals: { acase: acase, query: nil, query_count: query_count, counter: 0 } + locals: { acase: acase, query: nil, query_count: query_count, counter: -1 } ) fetch_service.complete diff --git a/app/models/snapshot.rb b/app/models/snapshot.rb index 4743c7a77..7f4b02f8c 100644 --- a/app/models/snapshot.rb +++ b/app/models/snapshot.rb @@ -30,11 +30,9 @@ class Snapshot < ApplicationRecord belongs_to :scorer, optional: true # shouldn't be optional! # see the call back delete_associated_objects for special delete logic. - # rubocop:disable Rails/HasManyOrHasOneDependent - has_many :snapshot_queries, dependent: :destroy - # rubocop:enable Rails/HasManyOrHasOneDependent + has_many :snapshot_queries, dependent: :destroy has_many :snapshot_docs, - through: :snapshot_queries, dependent: :destroy + through: :snapshot_queries, dependent: :destroy has_one_attached :snapshot_file @@ -43,7 +41,7 @@ class Snapshot < ApplicationRecord # Callbacks before_validation :set_defaults - #before_destroy :delete_associated_objects + # before_destroy :delete_associated_objects private @@ -52,7 +50,7 @@ def set_defaults end def delete_associated_objects - puts "I am about to delete associated objects" + puts 'I am about to delete associated objects' SnapshotDoc.joins(snapshot_query: :snapshot) .where(snapshot_queries: { snapshot: self }) .delete_all diff --git a/app/services/book_importer.rb b/app/services/book_importer.rb index 985f0f99d..39c5e7ea8 100644 --- a/app/services/book_importer.rb +++ b/app/services/book_importer.rb @@ -92,16 +92,23 @@ def import @book.save if params_to_use[:query_doc_pairs] - counter = params_to_use[:query_doc_pairs].size + total = params_to_use[:query_doc_pairs].size + counter = total + last_percent = 0 params_to_use[:query_doc_pairs].each do |query_doc_pair| qdp = @book.query_doc_pairs.create(query_doc_pair.except(:judgements)) counter -= 1 - Turbo::StreamsChannel.broadcast_render_to( - :notifications, - target: 'notifications', - partial: 'books/blah', - locals: { book: @book, counter: counter, qdp: qdp } - ) + # emit a message every percent that we cross, from 0 to 100... + percent = (((total - counter).to_f / total) * 100).truncate + if percent > last_percent + last_percent = percent + Turbo::StreamsChannel.broadcast_render_to( + :notifications, + target: 'notifications', + partial: 'books/blah', + locals: { book: @book, counter: counter, percent: percent, qdp: qdp } + ) + end next unless query_doc_pair[:judgements] query_doc_pair[:judgements].each do |judgement| diff --git a/app/services/fetch_service.rb b/app/services/fetch_service.rb index e61bf7e04..d45b67a10 100644 --- a/app/services/fetch_service.rb +++ b/app/services/fetch_service.rb @@ -51,9 +51,10 @@ def complete # Keep the first snapshot, and the most recents, deleting the ones out of the middle. # Not the best sampling! - snapshot_to_delete = @case.snapshots[1..((@options[:snapshot_limit] * -1) + @case.snapshots.count)] - # snapshot_to_delete&.each(&:destroy) - snapshot_to_delete.each(&:destroy!) + if @case.snapshots.count > @options[:snapshot_limit] + snapshots_to_delete = @case.snapshots[1..((@options[:snapshot_limit] * -1) + @case.snapshots.count)] + snapshots_to_delete.each(&:destroy!) + end end # rubocop:disable Metrics/MethodLength diff --git a/app/views/admin/query_runner/_notification.turbo_stream.erb b/app/views/admin/query_runner/_notification.turbo_stream.erb index 807dc83f6..5ef103aad 100644 --- a/app/views/admin/query_runner/_notification.turbo_stream.erb +++ b/app/views/admin/query_runner/_notification.turbo_stream.erb @@ -1,12 +1,15 @@ <%= turbo_stream.replace "notification" do %> - <% unless query.nil? %> - Query: <%=query.query_text%> - <% end %> - <% if counter > 0 %> -