Add Ubuntu controller (WIP).

deps.edn

@@ -28,6 +28,7 @@
 
         ;; exploratory
         com.hyperfiddle/rcf {:mvn/version "20220926-202227"}
+        org.suskalo/coffi {:mvn/version "1.0.486"}
         cloroutine/cloroutine {:local/root "/home/christian/Development/cloroutine"} #_{:mvn/version "13"}
         missionary/missionary {:mvn/version "b.41"}
         io.github.jbellis/jvector {:mvn/version "3.0.2"}
@@ -44,6 +45,7 @@
  :aliases
  {:dev {;:main-opts ["-m" "is.simm.simmis" "--middleware" "[cider.nrepl/cider-middleware]"]
         ;:jvm-opts ["--add-modules jdk.incubator.vector"]
+        :jvm-opts ["--enable-native-access=ALL-UNNAMED"]
         :extra-paths ["src-dev"]
         :extra-deps  {thheller/shadow-cljs          {:mvn/version "2.26.2"}
                       io.github.clojure/tools.build {:mvn/version "0.9.6"
@@ -54,10 +56,12 @@
                  {:mvn/version "0.9.6"}}
           :ns-default build
           ;:jvm-opts ["--add-modules jdk.incubator.vector"]
+          :jvm-opts ["--enable-native-access=ALL-UNNAMED"]
           }
   :test {:extra-paths ["test"]
          :extra-deps {org.clojure/test.check {:mvn/version "1.1.1"}
                       io.github.cognitect-labs/test-runner
                       {:git/tag "v0.5.1" :git/sha "dfb30dd"}}
          ;:jvm-opts ["--add-modules jdk.incubator.vector"]
+         :jvm-opts ["--enable-native-access=ALL-UNNAMED"]
          }}}

resources/prompts/imitate.txt

@@ -0,0 +1,4 @@
+Given the previous chat history, write only the next message that %s would write in their own style, language and character. Do not assist, imitate them as closely as possible.
+
+%s
+

resources/prompts/minecraft.txt

@@ -0,0 +1,30 @@
+You are a Minecraft player having fun in Minecraft. You get transcripts of the screen and are listening to the speakers. Go ahead and explore the world by pursuing meaningful goals.
+
+
+
+===== Recent audio input =====
+
+
+%s
+
+
+===== Recent screen descriptions =====
+
+
+%s 
+
+
+===== Your recent statements =====
+
+
+%s
+
+
+===== Your recent actions =====
+
+
+%s
+
+
+================
+

resources/prompts/screen.txt

@@ -0,0 +1,31 @@
+You are an computer assistant looking at the screen and listening to the speakers. You execute tasks on the screen.
+
+
+
+===== Recent audio input =====
+
+
+%s
+
+
+===== Recent screen descriptions =====
+
+
+%s 
+
+
+===== Your recent statements =====
+
+
+%s
+
+
+===== Your recent actions =====
+
+
+%s
+
+
+================
+
+

src-dev/user.clj

@@ -1 +1,5 @@
-(ns user (:require [dev]))
+(ns user 
+  (:require [dev]
+            [hyperfiddle.rcf]))
+
+(hyperfiddle.rcf/enable!)

src/is/simm/prompts.clj

@@ -9,16 +9,8 @@
 
 (def search (slurp (io/resource "prompts/search.txt")))
 
+(def imitate (slurp (io/resource "prompts/imitate.txt")))
 
-;; TODO unused, either factor out or remove
-(def clone-prompt "Given the previous chat history, write only the next message that %s would write in their own style, language and character. Do not assist, imitate them as closely as possible.\n\n%s\n\n")
-
-(def weather-affects-agenda "%s\n\n\nThis was the recent chat history. You are an assistant helping a user plan their day. You are given the following JSON describing the context.\n\n%s\n\nDoes the weather require special consideration for any of the agenda items? If not, just say NOOP! Otherwise keep explaining to %s directly, be brief.")
-
-(def emails-affect-agenda "%s\n\n\nThis was the recent chat history. You are an assistant helping a user plan their day. You are given the following JSON describing the context.\n\n%s\n\nDo any of the mails affect the agenda? If not, just say NOOP! Otherwise keep explaining to %s directly, be brief and only talk about the agenda.")
-
-(def agenda-change-needed "%s\n\nGiven the previous conversation, you need to decide whether changes to the agenda are needed. If not, answer with NOOP, else make a suggestion for how to change the agenda. Address %s directly.\n\n")
-
-
-
+(def screen (slurp (io/resource "prompts/screen.txt")))
 
+(def minecraft (slurp (io/resource "prompts/minecraft.txt")))

src/is/simm/runtimes/openai.clj

@@ -20,14 +20,13 @@
 
 (def api-key (:openai-key config))
 
-(defn encode-image [image-path]
-  (with-open [input-stream (io/input-stream image-path)]
-    (let [image-bytes (.readAllBytes input-stream)]
-      (.encodeToString (Base64/getEncoder) image-bytes))))
+(defn encode-file [file-path]
+  (with-open [input-stream (io/input-stream file-path)]
+    (let [file-bytes (.readAllBytes input-stream)]
+      (.encodeToString (Base64/getEncoder) file-bytes))))
 
 (def headers
-  {"Content-Type"  "application/json"
-   "Authorization" (str "Bearer " api-key)})
+  {"Authorization" (str "Bearer " api-key)})
 
 (defn payload [model content]
   (json/write-str
@@ -52,7 +51,7 @@
 (defn chat [model content]
   (let [res (promise-chan)
         cf (http/post "https://api.openai.com/v1/chat/completions"
-                      {:headers headers
+                      {:headers (assoc headers "Content-Type"  "application/json")
                        :body (payload model content)
                        :async true})]
     (-> cf
@@ -143,20 +142,76 @@
 
   )
 
+(comment
+  (defn stt [model input-path]
+    (let [audio-file ((py.- (py.- (py.- client audio) transcriptions) create) :model model :file ((py/path->py-obj "builtins.open") input-path "rb"))]
+      (py.- audio-file text)))
+  )
+
 (defn stt [model input-path]
-  (let [audio-file ((py.- (py.- (py.- client audio) transcriptions) create) :model model :file ((py/path->py-obj "builtins.open") input-path "rb"))]
-    (py.- audio-file text)))
+  (let [res (promise-chan)
+        request (http/post "https://api.openai.com/v1/audio/transcriptions"
+                           {:headers headers
+                            :multipart [{:name "file" :content (io/file input-path) :file-name input-path :mimetype "audio/wav"}
+                                        {:name "model" :content model}]
+                            :async true})]
+    (-> request
+        (.thenApply (reify Function
+                      (apply [_ response]
+                        (put! res
+                              (-> response
+                                  :body
+                                  json/read-str
+                                  (get "text"))))))
+        (.exceptionally (reify Function
+                          (apply [_ e]
+                            (put! res (ex-info "Error in OpenAI STT." {:type :error-in-openai :error e}))))))
+    res))
+
+(comment
+  (require '[missionary.core :as m])
+
+  (m/? (<! (stt-2 "whisper-1" "/tmp/microphone.wav")))
+
+  )
+
 
 (defn whisper-1 [input-path]
   (stt "whisper-1" input-path))
 
-(require-python '[pathlib :refer [Path]])
 
+
+
+(comment
+  (require-python '[pathlib :refer [Path]])
+
+  (defn tts-1 [text]
+    (let [res ((py.- (py.- (py.- client audio) speech) create) :model "tts-1" :voice "alloy" :input text)
+          rand-path (str "/tmp/" (java.util.UUID/randomUUID) ".mp3")]
+      ((py.- res stream_to_file) (Path rand-path))
+      rand-path))
+
+  )
+
+;; same but with http client
 (defn tts-1 [text]
-  (let [res ((py.- (py.- (py.- client audio) speech) create) :model "tts-1" :voice "alloy" :input text)
-        rand-path (str "/tmp/" (java.util.UUID/randomUUID) ".mp3")]
-    ((py.- res stream_to_file) (Path rand-path))
-    rand-path))
+  (let [res (promise-chan)
+        request (http/post "https://api.openai.com/v1/audio/speech"
+                           {:headers (assoc headers "Content-Type" "application/json")
+                            :body (json/write-str {:model "tts-1" :voice "alloy" :input text})
+                            :async true
+                            :as :stream})]
+    (-> request
+        (.thenApply (reify Function
+                      (apply [_ response]
+                        (let [rand-path (str "/tmp/" (java.util.UUID/randomUUID) ".mp3")]
+                          (io/copy (:body response) (io/file rand-path))
+                          (put! res rand-path)))))
+        (.exceptionally (reify Function
+                          (apply [_ e]
+                            (put! res (ex-info "Error in OpenAI TTS." {:type :error-in-openai :error e}))))))
+    res))
+
 
 (defn openai [[S peer [in out]]]
   (let [p (pub in (fn [{:keys [type]}]