lots of scaling fixes

Gemfile

@@ -18,6 +18,10 @@ gem 'd3-rails', '~> 3.5.5' # For cal heatmap
 gem 'devise', '>= 4.6.2'
 gem 'devise_invitable', '~> 2.0'
 
+gem 'active_storage_db'
+
+gem 'rubyzip'
+
 # Using this as it wires in via Sprockets and I can't get npm version to work with the main app.
 # Had no luck with js/svg approach ;-(
 gem 'font-awesome-sass'

Gemfile.lock

@@ -50,6 +50,9 @@ GEM
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
+    active_storage_db (1.3.0)
+      activestorage (>= 6.0)
+      rails (>= 6.0)
     activejob (7.1.2)
       activesupport (= 7.1.2)
       globalid (>= 0.3.6)
@@ -437,6 +440,7 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    stackprof (0.2.25)
     stringio (3.0.9)
     terser (1.1.19)
       execjs (>= 0.3.0, < 3)
@@ -477,6 +481,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  active_storage_db
   activerecord-import (>= 1.0.7)
   acts_as_list (>= 1.0.1)
   ancestry
@@ -528,10 +533,12 @@ DEPENDENCIES
   rubocop
   rubocop-capybara
   rubocop-rails
+  rubyzip
   sassc-rails (~> 2.1)
   selenium-webdriver
   sidekiq
   simplecov
+  stackprof
   terser
   thor
   turbolinks (~> 5)

app/controllers/api/v1/books/populate_controller.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
+require 'action_view'
+
 module Api
   module V1
     module Books
       class PopulateController < Api::ApiController
+        include ActionView::Helpers::NumberHelper
         before_action :find_book, only: [ :update ]
         before_action :check_book, only: [ :update ]
         before_action :find_case
@@ -12,46 +15,30 @@ class PopulateController < Api::ApiController
         # We get a messy set of params in this method, so we don't use the normal
         # approach of strong parameter validation.  We hardcode the only params
         # we care about.
-        # rubocop:disable Metrics/AbcSize
-        # rubocop:disable Metrics/MethodLength
         # rubocop:disable Layout/LineLength
         def update
-          # down the road we should be using ActiveRecord-import and first_or_initialize instead.
-          # See how snapshots are managed.
-
-          is_book_empty = @book.query_doc_pairs.empty?
-
-          params[:query_doc_pairs].each do |pair|
-            query_doc_pair = @book.query_doc_pairs.find_or_create_by query_text: pair[:query_text],
-                                                                     doc_id:     pair[:doc_id]
-            query_doc_pair.position = pair[:position]
-            query_doc_pair.document_fields = pair[:document_fields].to_json
-
-            query = @case.queries.find_by(query_text: query_doc_pair.query_text)
-
-            query_doc_pair.information_need = query.information_need
-            query_doc_pair.notes = query.notes
-            query_doc_pair.options = query.options
+          puts "[PopulateController] Request Size is #{number_to_human_size(query_doc_pairs_params.to_s.bytesize)}"
 
-            if pair[:rating]
-              rating = query.ratings.find_by(doc_id: query_doc_pair.doc_id)
+          serialized_data = Marshal.dump(query_doc_pairs_params)
 
-              # we are smart and just look up the correct user id from rating.user_id via the database, no API data needed.
-              judgement = query_doc_pair.judgements.find_or_create_by user_id: rating.user_id
-              judgement.rating = pair[:rating]
-              judgement.user = rating.user
-              judgement.save!
-            end
-
-            query_doc_pair.save!
-          end
-
-          Analytics::Tracker.track_query_doc_pairs_bulk_updated_event current_user, @book, is_book_empty
+          puts "[PopulateController] the size of the serialized data is #{number_to_human_size(serialized_data.bytesize)}"
+          compressed_data = Zlib::Deflate.deflate(serialized_data)
+          puts "[PopulateController] the size of the compressed data is #{number_to_human_size(compressed_data.bytesize)}"
+          @book.populate_file.attach(io: StringIO.new(compressed_data), filename: "book_populate_#{@book.id}.bin.zip",
+                                     content_type: 'application/zip')
+          PopulateBookJob.perform_later current_user, @book, @case
           head :no_content
         end
-        # rubocop:enable Metrics/AbcSize
-        # rubocop:enable Metrics/MethodLength
         # rubocop:enable Layout/LineLength
+
+        private
+
+        def query_doc_pairs_params
+          # avoid StrongParameters ;-( to faciliate sending params as
+          # hash to ActiveJob via ActiveStorage by directly getting parameters from request
+          # object
+          request.parameters
+        end
       end
     end
   end

app/controllers/api/v1/export/books_controller.rb

@@ -1,15 +1,17 @@
 # frozen_string_literal: true
 
-require 'zlib'
+require 'zip'
+require 'action_view'
+
 module Api
   module V1
     module Export
       class BooksController < Api::ApiController
+        include ActionView::Helpers::NumberHelper
         api!
         before_action :find_book
         before_action :check_book
 
-        # rubocop:disable Metrics/MethodLength
         # rubocop:disable Layout/LineLength
         def show
           # do we want to make the file downloadable
@@ -18,26 +20,23 @@ def show
           respond_to do |format|
             format.json do
               if download
-                # compressed_data = Zlib::Deflate.deflate(render_to_string(template: 'api/v1/export/books/show', locals: { book: @book }))
-                json_data = render_to_string(template: 'api/v1/export/books/show', locals: { book: @book })
+                json_data = render_to_string(template: 'api/v1/export/books/show')
 
-                puts 'about to attach'
-                @book.json_export.attach(io: StringIO.new(json_data), filename: "book_export_#{@book.id}.json",
-                                         content_type: 'application/json')
+                puts "the size of the json data is #{number_to_human_size(json_data.bytesize)}"
 
-                puts 'done with attach'
-                blob = @book.json_export.blob
-                puts 'about to geenrate url'
-                url = Rails.application.routes.url_helpers.rails_blob_url(blob, only_path: true)
-                puts "about to render ok with url: #{url}"
-                # render plain: "OK #{url}"
+                compressed_data = create_zip_from_json(json_data, "book_export_#{@book.id}.json")
+
+                @book.export_file.attach(io: compressed_data, filename: "book_export_#{@book.id}.json.zip",
+                                         content_type: 'application/zip')
+
+                blob = @book.export_file.blob
 
-                # send_data file_data, filename: blob.filename.to_s, type: blob.content_type, disposition: 'attachment'
+                url = Rails.application.routes.url_helpers.rails_blob_url(blob, only_path: true)
+                render json: { download_file_url: url }
               end
             end
           end
         end
-        # rubocop:enable Metrics/MethodLength
 
         private
 
@@ -49,6 +48,15 @@ def find_book
         def check_book
           render json: { message: 'Book not found!' }, status: :not_found unless @book
         end
+
+        def create_zip_from_json json_string, filename
+          zip_data = Zip::OutputStream.write_buffer do |zipfile|
+            zipfile.put_next_entry(filename)
+            zipfile.write(json_string)
+          end
+          zip_data.rewind
+          zip_data
+        end
       end
     end
   end

app/controllers/api/v1/judgements_controller.rb

@@ -3,23 +3,47 @@
 module Api
   module V1
     class JudgementsController < Api::ApiController
-      api!
       before_action :find_book
       before_action :check_book
       before_action :set_judgement,   only: [ :show, :update, :destroy ]
       before_action :check_judgement, only: [ :show, :update, :destroy ]
 
+      def_param_group :judgement do
+        param :rating, Float
+        param :judge_later, [ true, false ]
+        param :unrateable, [ true, false ]
+        param :user_id, String
+      end
+
+      api :GET, '/api/books/:book_id/judgements',
+          'List all judgements for the book'
+      param :book_id, :number,
+            desc: 'The ID of the requested book.', required: true
       def index
         @judgements = @book.judgements
 
         respond_with @judgements
       end
 
+      api :GET, '/api/books/:book_id/query_doc_pairs/:query_doc_pair_id/judgements/:id',
+          'Show the judgement with the given ID.'
+      param :book_id, :number,
+            desc: 'The ID of the requested book.', required: true
+      param :query_doc_pair_id, :number,
+            desc: 'The ID of the requested query doc pair.', required: true
+      param :id, :number,
+            desc: 'The ID of the requested judgement.', required: true
       def show
         respond_with @judgement
       end
 
       # rubocop:disable Metrics/AbcSize
+      api :POST, '/api/books/:book_id/query_doc_pair/:query_doc_pair_id/judgements/', 'Create a new judgement.'
+      param :query_doc_pair_id, :number,
+            desc: 'The ID of the requested query doc pair.', required: true
+      param :book_id, :number,
+            desc: 'The ID of the requested book.', required: true
+      param_group :judgement
       def create
         # @judgement = @book.judgements.build judgement_params
         @judgement = @book.judgements.find_or_create_by query_doc_pair_id: params[:judgement][:query_doc_pair_id],
@@ -38,8 +62,16 @@ def create
           render json: @judgement.errors, status: :bad_request
         end
       end
-      # rubocop:enable Metrics/AbcSize
 
+      # rubocop:enable Metrics/AbcSize
+      api :PUT, '/api/books/:book_id/query_doc_pair/:query_doc_pair_id/judgements/:id', 'Update a given judgement.'
+      param :book_id, :number,
+            desc: 'The ID of the requested book.', required: true
+      param :query_doc_pair_id, :number,
+            desc: 'The ID of the requested query doc pair.', required: true
+      param :id, :number,
+            desc: 'The ID of the requested judgement.', required: true
+      param_group :judgement
       def update
         update_params = judgement_params
         if @judgement.update update_params
@@ -49,6 +81,13 @@ def update
         end
       end
 
+      api :DELETE, '/api/books/:book_id/query_doc_pair/:query_doc_pair_id/judgements/:id', 'Delete a given judgement.'
+      param :book_id, :number,
+            desc: 'The ID of the requested book.', required: true
+      param :query_doc_pair_id, :number,
+            desc: 'The ID of the requested query doc pair.', required: true
+      param :id, :number,
+            desc: 'The ID of the judgement.', required: true
       def destroy
         @judgement.destroy
         head :no_content

app/controllers/api/v1/query_doc_pairs_controller.rb

@@ -24,8 +24,8 @@ class QueryDocPairsController < Api::ApiController
       param :id, :number,
             desc: 'The ID of the requested book.', required: true
       def index
-        @query_doc_pairs = @book.query_doc_pairs.includes([ :judgements ])
-
+        # @query_doc_pairs = @book.query_doc_pairs.includes([ :judgements ])
+        @query_doc_pairs = @book.query_doc_pairs
         respond_with @query_doc_pairs
       end
 

app/controllers/books/import_controller.rb

@@ -2,19 +2,24 @@
 
 require 'open-uri'
 require 'json'
+
+require 'action_view'
+
 module Books
   class ImportController < ApplicationController
+    include ActionView::Helpers::NumberHelper
     def new
       @book = Book.new
     end
 
     # rubocop:disable Metrics/MethodLength
     # rubocop:disable Security/Open
+    # rubocop:disable Metrics/AbcSize
     def create
       @book = Book.new
       @book.owner = current_user
 
-      uploaded_file = params[:book][:json_upload]
+      uploaded_file = params[:book][:import_file]
       json_file = uploaded_file.tempfile
       json_data = URI.open(json_file) do |file|
         JSON.parse(file.read)
@@ -24,7 +29,6 @@ def create
         params_to_use = JSON.parse(json_data.read).deep_symbolize_keys
 
         @book.name = params_to_use[:name]
-        @book.json_upload.attach(uploaded_file)
 
         service = ::BookImporter.new @book, params_to_use, {}
         service.validate
@@ -33,6 +37,13 @@ def create
       end
 
       if @book.errors.empty? && @book.save
+        serialized_data = Marshal.dump(params_to_use)
+        puts "the size of the serialized data is #{number_to_human_size(serialized_data.bytesize)}"
+        compressed_data = Zlib::Deflate.deflate(serialized_data)
+        puts "the size of the compressed data is #{number_to_human_size(compressed_data.bytesize)}"
+        @book.import_file.attach(io: StringIO.new(compressed_data), filename: "book_import_#{@book.id}.bin.zip",
+                                 content_type: 'application/zip')
+        @book.save
         ImportBookJob.perform_later @book
         redirect_to @book, notice: 'Book was successfully created.'
       else
@@ -41,5 +52,6 @@ def create
     end
     # rubocop:enable Metrics/MethodLength
     # rubocop:enable Security/Open
+    # rubocop:enable Metrics/AbcSize
   end
 end

app/jobs/import_book_job.rb

@@ -3,16 +3,19 @@
 class ImportBookJob < ApplicationJob
   queue_as :default
 
+  # rubocop:disable Security/MarshalLoad
   def perform book
     options = {}
+    pp book.import_file
+    compressed_data = book.import_file.download
+    serialized_data = Zlib::Inflate.inflate(compressed_data)
+    params  = Marshal.load(serialized_data)
 
-    # Read and parse the JSON data
-    json_data = JSON.parse(book.json_upload.download)
-
-    service = ::BookImporter.new book, json_data.deep_symbolize_keys, options
+    service = ::BookImporter.new book, params, options
 
     service.import
-    book.json_upload.purge
+    book.import_file.purge
     book.save
   end
+  # rubocop:enable Security/MarshalLoad
 end