Jupyterlite Support in Quepid (#544)

Ship one real notebook for comparing snapshots.

.gitignore

@@ -45,6 +45,11 @@ coverage/
 # Docker artifacts
 public/srv
 
+# Jupyterlite
+public/notebooks
+public/notebooks-*
+.jupyterlite.doit.db
+
 # Test Reports
 test/reports
 

Dockerfile.dev

@@ -29,5 +29,10 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x | bash -
 RUN apt-get update -qq && apt-get install -y --no-install-recommends nodejs yarn netcat \
   && rm -rf /var/lib/apt/lists/*
 
+# Jupyterlite
+RUN apt-get update -qq && apt-get install -y python3 python3-pip && rm -rf /var/lib/apt/lists/*
+RUN python3 -m pip install jupyterlite[lab] jupyterlab-webrtc-docprovider
+
+
 # Clean environment
 RUN apt-get clean all

README.md

@@ -105,7 +105,7 @@ This is useful for stress testing Quepid!  Especially the front end application!
 
 #### 3. Running the app
 
-Now fire up Quepid locally at http://localhost:3000:
+Now fire up Quepid locally at http://localhost:
 
 ```
 bin/docker server
@@ -541,6 +541,13 @@ For the various Admin pages, we actually are using Bootstrap 5! That is included
 We currently use Rails Sprockets to compile everything, but do have dreams of moving the JavaScript
 over to Webpacker.
 
+## I'd like to develop Jupyterlite
+
+Run the `./bin/setup_jupyterlite` to update the archive file `./jupyterlite/notebooks.gz`.  This
+also sets up the static files in the `./public/notebooks` directory.  However, so we don't check in hundreds of files,
+we ignore that directory.   At `asset:precompile` time we unpack the `./jupyterlite/notebooks.gz` file instead.  This
+works on Heroku and the production Docker image.
+
 # QA
 
 There is a code deployment pipeline to the http://quepid-staging.herokuapp.com site that

app/views/api/v1/snapshots/_snapshot.json.jbuilder

@@ -13,12 +13,12 @@ end
 
 if with_docs
   json.docs do
-    snapshot.snapshot_queries.each do |query|
-      docs = query.snapshot_docs.map do |doc|
+    snapshot.snapshot_queries.each do |snapshot_query|
+      docs = snapshot_query.snapshot_docs.map do |doc|
         { id: doc.doc_id, explain: doc.explain, rated_only: doc.rated_only }
       end
 
-      json.set! query.query_id, docs
+      json.set! snapshot_query.query_id, docs
     end
   end
 end
@@ -29,3 +29,9 @@ if with_docs
     json.array! snapshot.snapshot_queries.collect(&:query).compact, partial: 'api/v1/queries/query', as: :query
   end
 end
+
+unless shallow
+  json.scores do
+    json.array! snapshot.snapshot_queries, partial: 'api/v1/snapshots/snapshot_query', as: :snapshot_query
+  end
+end

app/views/api/v1/snapshots/_snapshot_query.json.jbuilder

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+json.queryId      snapshot_query.query.id
+json.score        snapshot_query.score
+json.all_rated    snapshot_query.all_rated

app/views/layouts/_header_core_app.html.erb

@@ -44,6 +44,7 @@
 
           <li><a ng-attr-target="{{ isRailsGoingToAngular() ? '_self' : undefined }}" href="/teams">Teams</a></li>
           <li><a ng-attr-target="{{ isRailsGoingToAngular() ? '_self' : undefined }}" href="/scorers">Scorers</a></li>
+          <li><a ng-attr-target="_blank" href="/notebooks/lab/index.html">Notebooks</a></li>
         </ul>
       <% end %>
       <ul class="navbar-right nav navbar-nav">

bin/setup_docker

@@ -35,7 +35,7 @@ Dir.chdir APP_ROOT do
   system "bin/docker r ./wait-for mysql:3306 -- bin/rake db:setup"
   system "bin/docker r bin/rake db:migrate"
   system "bin/docker r ./wait-for mysql:3306 -- bin/rake db:create RAILS_ENV=test"
-  
+
   puts "\n== Setting up Seed Data for Development =="
   system "bin/docker r bin/rake db:seed:sample_users"
 end

bin/setup_jupyterlite

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+# Update our Jupyterlite setup from the latest.
+
+# path to your application root.
+APP_ROOT = Pathname.new File.expand_path('../../',  __FILE__)
+
+Dir.chdir APP_ROOT do
+
+  puts "\n== Setting up Jupyter Lite Notebooks =="
+  system "bin/docker r mv /srv/app/public/notebooks /srv/app/public/notebooks-$(date +%s)"
+  system "bin/docker r jupyter lite build  --lite-dir /srv/app/jupyterlite --apps=lab --no-sourcemaps --contents ./jupyterlite/files --output-dir /srv/app/public/notebooks"
+  system "bin/docker r tar -czf /srv/app/jupyterlite/notebooks.gz /srv/app/public/notebooks"
+
+end

config/routes.rb

@@ -166,6 +166,6 @@
   get '/scorers'                      => 'core#index'
 
   # Static pages
-  get '*page' => 'pages#show'
+  # get '*page' => 'pages#show'
 end
 # rubocop:enable Metrics/BlockLength

docs/jupyterlite.md

@@ -0,0 +1,21 @@
+https://github.com/innovationOUtside/ouseful_jupyterlite_utils
+
+From https://jupyterlite.readthedocs.io/en/latest/user-guide.html#how-can-i-read-content-from-python
+
+```
+import pandas as pd
+from js import fetch
+
+URL = "https://yourdomain.com/path/to/file.csv"
+
+res = await fetch(URL)
+text = await res.text()
+
+filename = 'data.csv'
+
+with open(filename, 'w') as f:
+    f.write(text)
+
+data = pd.read_csv(filename, sep=';')
+data
+```

jupyterlite/files/README.md

@@ -0,0 +1,5 @@
+# Welcome to the Jupyterlite based notebooks
+
+Learn more at https://jupyterlite.readthedocs.io/en/latest/
+
+* ./examples/Snapshot Compare.ipynb is an example of measuring baseline relevancy over time.

jupyterlite/jupyter-lite.json

@@ -0,0 +1,5 @@
+{
+  "jupyter-config-data": {
+    "collaborative": true
+  }
+}

lib/tasks/assets.rake

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
+require 'rubygems/package'
+require 'zlib'
+
+# rubocop:disable Metrics/BlockLength
+
+GZIPPED_BASE_PATH_FOR_NOTEBOOKS = '/srv/app/public/notebooks'
 namespace :assets do
+  # Does this actually do anything for us?
   desc 'Create .gz versions of assets'
   task gzip: :environment do
     zip_types = /\.(?:css|html|js|otf|svg|txt|xml)$/
@@ -25,8 +32,40 @@ namespace :assets do
     end
   end
 
+  desc 'Unpack Jupyterlite assets'
+  task jupyterlite: :environment do
+    notebooks_zip = Rails.root.join('jupyterlite/notebooks.gz')
+    destination = Rails.public_path.join('notebooks')
+
+    Gem::Package::TarReader.new( Zlib::GzipReader.open(notebooks_zip) ) do |tar|
+      dest = nil
+      tar.each do |entry|
+        entry_path = entry.full_name
+        entry_path = entry_path[GZIPPED_BASE_PATH_FOR_NOTEBOOKS.length..]
+
+        dest ||= File.join destination, entry_path
+        if entry.directory?
+          FileUtils.rm_rf dest unless File.directory? dest
+          FileUtils.mkdir_p dest, mode: entry.header.mode, verbose: false
+        elsif entry.file?
+          FileUtils.rm_rf dest unless File.file? dest
+          File.open dest, 'wb' do |f|
+            f.print entry.read
+          end
+          FileUtils.chmod entry.header.mode, dest, verbose: false
+        elsif '2' == entry.header.typeflag # Symlink!
+          File.symlink entry.header.linkname, dest
+        end
+        dest = nil
+      end
+    end
+  end
+
   # Hook into existing assets:precompile task
   Rake::Task['assets:precompile'].enhance do
     Rake::Task['assets:gzip'].invoke
+    Rake::Task['assets:jupyterlite'].invoke
   end
 end
+
+# rubocop:enable Metrics/BlockLength

test/controllers/core_controller_test.rb

@@ -18,7 +18,6 @@ class CoreControllerTest < ActionController::TestCase
     test 'should get index' do
       get :index
       assert_response :success
-      puts "Request is #{request.ssl?}"
     end
   end