Merge branch 'main' into chakra-testing

athensresearch · Apr 12, 2022 · 6336a9e · 6336a9e
2 parents 61ca62e + 8a510f8
commit 6336a9e
Show file tree

Hide file tree

Showing 17 changed files with 467 additions and 191 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,89 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [2.0.0-beta.27](https://github.com/athensresearch/athens/compare/v2.0.0-beta.25...v2.0.0-beta.27) (2022-04-07)
+
+
+### Features
+
+* Don't report PII ([d230732](https://github.com/athensresearch/athens/commit/d2307328c35c8c0770dfb7ed89ea379926406978))
+* migrate events without a uid ([98ab9df](https://github.com/athensresearch/athens/commit/98ab9dfe16556b293120a53f064c2365d85e46b8))
+* migrate to efficient event log filtering ([420a128](https://github.com/athensresearch/athens/commit/420a1286627e22a9d978f936001a7c6e10443566))
+* support event log migrations ([47fcabe](https://github.com/athensresearch/athens/commit/47fcabe2371ec21e641058ea88e5e6b04ba23f41))
+
+
+### Bug Fixes
+
+* allow remote dbs in web athens ([094192e](https://github.com/athensresearch/athens/commit/094192e5d23a821f0f04005e707310c082016f4e))
+* bugs alex found ([677fb6e](https://github.com/athensresearch/athens/commit/677fb6ea085b4ed1d254d2e41b765b68a559c75e))
+* current version is 0 if none is present ([bb8ef68](https://github.com/athensresearch/athens/commit/bb8ef68b886efe07f32ccfe5ce60de20c5c934c3))
+* db-dump also ignored from limit on received ([a91df3c](https://github.com/athensresearch/athens/commit/a91df3cd60c05d132b008cf3fff21c1eca0dc483))
+* event-log/events now received id as kw arg ([25543ff](https://github.com/athensresearch/athens/commit/25543ff4a23df89b86e13ae83cd70adb98c72c1a))
+* exclude db dump from size limit ([7b28172](https://github.com/athensresearch/athens/commit/7b28172aaf764ea98b214ffe910eaafb211d8bfe))
+* Feature block link correct counting ([#2111](https://github.com/athensresearch/athens/issues/2111)) ([f1807a2](https://github.com/athensresearch/athens/commit/f1807a2b35879b281c22e46b52dd8ba86996b7fc))
+* fix args on migrate call site ([58ae736](https://github.com/athensresearch/athens/commit/58ae736581c59805305cc8f611af55f59c7d18e0))
+* fix wording on some limit messages ([7f55150](https://github.com/athensresearch/athens/commit/7f55150b1ec395740bb859250fd0c2799ad7113e))
+* get-current-version should return 0 on err ([a94fc64](https://github.com/athensresearch/athens/commit/a94fc648ce5d70d3eec78ba3f36e5b3a84dcb4d7))
+* minor migration logging ([d716ae7](https://github.com/athensresearch/athens/commit/d716ae7d49581d9b766780091422c79c9ab1ff5e))
+* query page size for migrations should be 100 ([7dadf54](https://github.com/athensresearch/athens/commit/7dadf546142095474e107884d573672d748040c5))
+* remove leftover print ([84624c3](https://github.com/athensresearch/athens/commit/84624c3cf1b217706f381448fde3076cb12b49bb))
+* set a 1MB event limit ([81198f4](https://github.com/athensresearch/athens/commit/81198f408119df7fa288fe0a1d042ba8bf8e0be1))
+* show remote db page in db modal when not electron ([477ecd1](https://github.com/athensresearch/athens/commit/477ecd1b28377127415d6d0fe6449d8f3557945d))
+* transit usage was very bork in clj ([92210e2](https://github.com/athensresearch/athens/commit/92210e2d260700828bc849e1324fe98c7fe930c9))
+* use long instead of bigint for event log ([20ee4b4](https://github.com/athensresearch/athens/commit/20ee4b482e260425db8ffd491c6a73bd7c8e2a6a))
+
+
+### Refactors
+
+* add athens.self-hosted.fluree.utils ([19bf9d3](https://github.com/athensresearch/athens/commit/19bf9d338a41794eb588c12f9c6fca7467e58d20))
+* move migrator into own ns ([0f97365](https://github.com/athensresearch/athens/commit/0f97365e97f422669f61c2b176646309f97541fd))
+
+
+* ignore unused test helper ([146f459](https://github.com/athensresearch/athens/commit/146f459bbd8b5b3100c6dd88a7d07a49afb477b1))
+* run fluree tests manually for now ([bb033c8](https://github.com/athensresearch/athens/commit/bb033c8ae1c4585ab03d36e21e530f70cc5c8172))
+* update to clojure 11 ([8ac293e](https://github.com/athensresearch/athens/commit/8ac293ea4511e54c9400081f4704d2d688638833))
+
+## [2.0.0-beta.26](https://github.com/athensresearch/athens/compare/v2.0.0-beta.25...v2.0.0-beta.26) (2022-04-06)
+
+
+### Features
+
+* migrate events without a uid ([98ab9df](https://github.com/athensresearch/athens/commit/98ab9dfe16556b293120a53f064c2365d85e46b8))
+* migrate to efficient event log filtering ([420a128](https://github.com/athensresearch/athens/commit/420a1286627e22a9d978f936001a7c6e10443566))
+* support event log migrations ([47fcabe](https://github.com/athensresearch/athens/commit/47fcabe2371ec21e641058ea88e5e6b04ba23f41))
+
+
+### Bug Fixes
+
+* allow remote dbs in web athens ([094192e](https://github.com/athensresearch/athens/commit/094192e5d23a821f0f04005e707310c082016f4e))
+* bugs alex found ([677fb6e](https://github.com/athensresearch/athens/commit/677fb6ea085b4ed1d254d2e41b765b68a559c75e))
+* current version is 0 if none is present ([bb8ef68](https://github.com/athensresearch/athens/commit/bb8ef68b886efe07f32ccfe5ce60de20c5c934c3))
+* db-dump also ignored from limit on received ([a91df3c](https://github.com/athensresearch/athens/commit/a91df3cd60c05d132b008cf3fff21c1eca0dc483))
+* event-log/events now received id as kw arg ([25543ff](https://github.com/athensresearch/athens/commit/25543ff4a23df89b86e13ae83cd70adb98c72c1a))
+* exclude db dump from size limit ([7b28172](https://github.com/athensresearch/athens/commit/7b28172aaf764ea98b214ffe910eaafb211d8bfe))
+* Feature block link correct counting ([#2111](https://github.com/athensresearch/athens/issues/2111)) ([f1807a2](https://github.com/athensresearch/athens/commit/f1807a2b35879b281c22e46b52dd8ba86996b7fc))
+* fix args on migrate call site ([58ae736](https://github.com/athensresearch/athens/commit/58ae736581c59805305cc8f611af55f59c7d18e0))
+* fix wording on some limit messages ([7f55150](https://github.com/athensresearch/athens/commit/7f55150b1ec395740bb859250fd0c2799ad7113e))
+* get-current-version should return 0 on err ([a94fc64](https://github.com/athensresearch/athens/commit/a94fc648ce5d70d3eec78ba3f36e5b3a84dcb4d7))
+* minor migration logging ([d716ae7](https://github.com/athensresearch/athens/commit/d716ae7d49581d9b766780091422c79c9ab1ff5e))
+* query page size for migrations should be 100 ([7dadf54](https://github.com/athensresearch/athens/commit/7dadf546142095474e107884d573672d748040c5))
+* remove leftover print ([84624c3](https://github.com/athensresearch/athens/commit/84624c3cf1b217706f381448fde3076cb12b49bb))
+* set a 1MB event limit ([81198f4](https://github.com/athensresearch/athens/commit/81198f408119df7fa288fe0a1d042ba8bf8e0be1))
+* show remote db page in db modal when not electron ([477ecd1](https://github.com/athensresearch/athens/commit/477ecd1b28377127415d6d0fe6449d8f3557945d))
+* transit usage was very bork in clj ([92210e2](https://github.com/athensresearch/athens/commit/92210e2d260700828bc849e1324fe98c7fe930c9))
+* use long instead of bigint for event log ([20ee4b4](https://github.com/athensresearch/athens/commit/20ee4b482e260425db8ffd491c6a73bd7c8e2a6a))
+
+
+### Refactors
+
+* add athens.self-hosted.fluree.utils ([19bf9d3](https://github.com/athensresearch/athens/commit/19bf9d338a41794eb588c12f9c6fca7467e58d20))
+* move migrator into own ns ([0f97365](https://github.com/athensresearch/athens/commit/0f97365e97f422669f61c2b176646309f97541fd))
+
+
+* ignore unused test helper ([146f459](https://github.com/athensresearch/athens/commit/146f459bbd8b5b3100c6dd88a7d07a49afb477b1))
+* run fluree tests manually for now ([bb033c8](https://github.com/athensresearch/athens/commit/bb033c8ae1c4585ab03d36e21e530f70cc5c8172))
+* update to clojure 11 ([8ac293e](https://github.com/athensresearch/athens/commit/8ac293ea4511e54c9400081f4704d2d688638833))
+
 ## [2.0.0-beta.25](https://github.com/athensresearch/athens/compare/v2.0.0-beta.24...v2.0.0-beta.25) (2022-04-04)
 
 

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "Athens",
   "author": "athensresearch",
-  "version": "2.0.0-beta.25",
+  "version": "2.0.0-beta.27",
   "description": "An open-source knowledege graph for research and notetaking",
   "repository": {
     "type": "git",

diff --git a/resources/public/index.html b/resources/public/index.html
@@ -10,7 +10,13 @@
       </noscript>
       <script>
         !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.async=!0,p.src=s.api_host+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="capture identify alias people.set people.set_once set_config register register_once unregister opt_out_capturing has_opted_out_capturing opt_in_capturing reset isFeatureEnabled onFeatureFlags".split(" "),n=0;n<o.length;n++)g(u,o[n]);e._i.push([i,s,a])},e.__SV=1)}(document,window.posthog||[]);
-        posthog.init('qouvHPR2-blOal8-l6CSmFybj8VWojBlv472Ac-ovAI',{api_host:'https://app.posthog.com', persistence:'localStorage', autocapture:false})
+        posthog.init('qouvHPR2-blOal8-l6CSmFybj8VWojBlv472Ac-ovAI',
+                     {api_host:'https://app.posthog.com',
+                      persistence:'localStorage',
+                      autocapture:false,
+                      property_blacklist:['$ip', /* it's not getting filtered out */
+                                          '$current_url',
+                                          '$pathname']})
       </script>
   </head>
   <body>

diff --git a/src/clj/athens/self_hosted/clients.clj b/src/clj/athens/self_hosted/clients.clj
@@ -4,34 +4,14 @@
     [athens.common-events        :as common-events]
     [athens.common-events.schema :as schema]
     [athens.common.logging       :as log]
-    [cognitect.transit           :as transit]
-    [org.httpkit.server          :as http])
-  (:import
-    (java.io
-      ByteArrayInputStream
-      ByteArrayOutputStream)))
+    [org.httpkit.server          :as http]))
 
 
 ;; Internal state
 ;; channel -> session info
 (defonce clients (atom {}))
 
 
-(defn ->transit
-  [data]
-  (let [out    (ByteArrayOutputStream. 4096)
-        writer (transit/writer out :json)]
-    (transit/write writer data)
-    (.toString out)))
-
-
-(defn <-transit
-  [transit-str]
-  (let [in (ByteArrayInputStream. (.getBytes transit-str))
-        reader (transit/reader in :json)]
-    (transit/read reader)))
-
-
 ;; Client management API
 
 (defn get-client-session
@@ -71,14 +51,24 @@
         valid-server-event?   (schema/valid-server-event? data)]
     (if (or valid-event-response?
             valid-server-event?)
-      (let [type   (common-events/find-event-or-atomic-op-type data)
-            status (:event/status data)]
-        (log/debug "Sending to username:" username
-                   ", event-id:" (:event/id data)
-                   (if type
-                     (str ", type: " type)
-                     (str ", status: " status)))
-        (http/send! channel (->transit data)))
+      (let [type             (common-events/find-event-or-atomic-op-type data)
+            status           (:event/status data)
+            serialized-event (common-events/serialize data)
+            errors           (when-not (common-events/ignore-serialized-event-validation? data)
+                               (common-events/validate-serialized-event serialized-event))]
+        (if errors
+          (log/error "Not sending invalid event to username:" username
+                     ", event-id:" (:event/id data)
+                     ", type:" (common-events/find-event-or-atomic-op-type data)
+                     ", invalid serialized event:"
+                     "event-response take:" (str errors))
+          (do
+            (log/debug "Sending to username:" username
+                       ", event-id:" (:event/id data)
+                       (if type
+                         (str ", type: " type)
+                         (str ", status: " status)))
+            (http/send! channel serialized-event))))
       ;; TODO internal failure mode, collect in reporting
       (log/error "Not sending invalid event to username:" username
                  ", event-id:" (:event/id data)

diff --git a/src/clj/athens/self_hosted/components/web.clj b/src/clj/athens/self_hosted/components/web.clj
@@ -59,13 +59,28 @@
   (fn receive-handler
     [channel msg]
     (let [username (clients/get-client-username channel)
-          data     (clients/<-transit msg)]
-      (if-not (schema/valid-event? data)
+          data     (common-events/deserialize msg)
+          errors   (common-events/validate-serialized-event msg)]
+      (cond
+        ;; TODO: we should be able to validate the serialized event without deserializing it, but
+        ;; we also need to build a rejected event to pass back to the client, and to do that we need
+        ;; the :event/id, which we only get after deserializing the event.
+        ;; I think this means that we should separate the :event/id from the payload itself.
+        errors
+        (do
+          (log/warn "username:" username "Invalid serialized event received, errors:" errors)
+          (clients/send! channel (common-events/build-event-rejected (:event/id data)
+                                                                     (str "Invalid serialized event")
+                                                                     errors)))
+
+        (not (schema/valid-event? data))
         (let [explanation (schema/explain-event data)]
           (log/warn "username:" username "Invalid event received, explanation:" explanation)
           (clients/send! channel (common-events/build-event-rejected (:event/id data)
                                                                      (str "Invalid event: " (pr-str data))
                                                                      explanation)))
+
+        :else
         (let [{:event/keys [id type]} data]
           (log/info "Received valid event" "username:" username ", event-id:" id ", type:" (common-events/find-event-or-atomic-op-type data))
           (let [{:event/keys [status]

diff --git a/src/cljc/athens/common_events.cljc b/src/cljc/athens/common_events.cljc
@@ -1,7 +1,85 @@
 (ns athens.common-events
   "Event as Verbs executed on Knowledge Graph"
   (:require
-    [athens.common.utils :as utils]))
+    [athens.common.utils :as utils]
+    [cognitect.transit :as transit]
+    #?(:cljs [com.cognitect.transit.types :as ty]))
+  #?(:clj
+     (:import
+       (java.io
+         ByteArrayInputStream
+         ByteArrayOutputStream))))
+
+
+;; Limits
+
+;; Fluree default max size over websocket is ~2mb.
+;; There doesn't seem to be a max for nginx
+;; https://serverfault.com/questions/1034906/can-nginx-limit-incoming-websocket-message-size
+;; Was able to transmit 500mb over websocket from the server to client.
+;; Let's settle on a nice sensible 1MB limit for now.
+(def max-event-size-in-bytes (* 1 1000 1000))
+
+
+(defn valid-serialized-event?
+  [serialized-event]
+  (< (count serialized-event) max-event-size-in-bytes))
+
+
+(defn validate-serialized-event
+  [serialized-event]
+  (when-not (valid-serialized-event? serialized-event)
+    (ex-info "Serialized event is larger than 1 MB" {})))
+
+
+(defn ignore-serialized-event-validation?
+  [event]
+  (-> event :event/type
+      ;; db-dump is sending the whole database and can (easily) go over max-event-size-in-bytes.
+      ;; Only real solution for this is to break down the db-dump into smaller pieces,
+      ;; possibly transitioning to partial loading by default in the future.
+      #{:datascript/db-dump}))
+
+
+;; serialization and limits
+
+;; Really shouldn't need these UUID and datom-reader, but we still send datoms via db-dump.
+#?(:cljs
+   ;; see https://github.com/cognitect/transit-cljs/issues/41#issuecomment-503287258
+   (extend-type ty/UUID IUUID))
+
+
+(def ^:private datom-reader
+  (transit/read-handler
+    (fn [[e a v tx added]]
+      {:e     e
+       :a     a
+       :v     v
+       :tx    tx
+       :added added})))
+
+
+(def serialization-type :json)
+(def serialization-opts {:handlers {:datom datom-reader}})
+
+
+(defn serialize
+  [event]
+  #?(:cljs (-> (transit/writer serialization-type)
+               (transit/write event))
+     :clj  (let [out    (ByteArrayOutputStream. 4096)
+                 writer (transit/writer out serialization-type)]
+             (transit/write writer event)
+             (.toString out))))
+
+
+(defn deserialize
+  [serialized-event]
+  #?(:cljs (-> (transit/reader serialization-type serialization-opts)
+               (transit/read serialized-event))
+     :clj  (let [in     (ByteArrayInputStream. (.getBytes serialized-event))
+                 reader (transit/reader in serialization-type serialization-opts)]
+             (transit/read reader))))
 
 
 ;; building events