Skip to content

Commit

Permalink
Bump deps, fix browing by swapping to firefox.
Browse files Browse the repository at this point in the history
  • Loading branch information
whilo committed Sep 21, 2024
1 parent 2d61d41 commit c8db31b
Show file tree
Hide file tree
Showing 6 changed files with 232 additions and 22 deletions.
8 changes: 5 additions & 3 deletions deps.edn
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{:paths ["src" "resources"]
:deps {org.clojure/clojure {:mvn/version "1.11.1"}
io.replikativ/datahike {:mvn/version "0.6.1558"}
io.replikativ/datahike {:mvn/version "0.6.1576"}
clj-python/libpython-clj {:mvn/version "2.025"}
morse/morse {:mvn/version "0.4.3"}
io.replikativ/kabel {:mvn/version "0.2.2"}
Expand All @@ -10,10 +10,12 @@
hiccup/hiccup {:mvn/version "2.0.0-RC3"}
org.clj-commons/hickory {:mvn/version "0.7.4"}
ring/ring-jetty-adapter {:mvn/version "1.12.0"}
etaoin/etaoin {:mvn/version "1.0.40"}
missionary/missionary {:mvn/version "b.34"}
etaoin/etaoin {:mvn/version "1.1.41"}
nrepl/nrepl {:mvn/version "1.1.1"}
cider/cider-nrepl {:mvn/version,"0.47.1"}

;; exploratory
missionary/missionary {:mvn/version "b.34"}
pangloss/pattern {:git/url "https://github.com/pangloss/pattern"
:sha "affc7f3ac907f5b98de6638574a741e4693f1648"}
anglican/anglican {:mvn/version "1.1.0"}}
Expand Down
119 changes: 119 additions & 0 deletions src/ie/simm/processes/build_accounting.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
(ns ie.simm.processes.build-accounting
(:require [clojure.core.async :as async :refer [timeout]]
[superv.async :refer [go-try go-loop-try <? S]]
[clojure.string :as str]
[ie.simm.runtimes.openai :as openai]
[ie.simm.runtimes.brave :as brave]
[ie.simm.runtimes.etaoin :as etaoin]
[anglican.core :as anglican]
[clojure.java.io :as io]
[hiccup2.core :as h]
[datahike.api :as d]
[morse.polling :as p]))

(-> "accounting system"
brave/search-brave
brave/extract-url
etaoin/extract-body)

(etaoin/extract-body "https://en.wikipedia.org/wiki/Accounting")


(defn interleave-with-delimiters
([coll] (interleave-with-delimiters coll "=========================================\n"))
([coll delimiter]
(str/join "\n" (interleave coll (repeat delimiter)))))


(defn feedback [question]
(println question)
(read-line))

(defn expert [type question]
(openai/chat "gpt-4o" (interleave-with-delimiters [(format "You are an %s." type) question])))

(defn critic [type question answer]
(Integer/parseInt
(openai/chat "gpt-4o" (interleave-with-delimiters [(format "You are a critic for a %s." type)
"We gave the expert this question:" question
"The expert replied with this answer:" answer
"Rate the expert's answer on a scale from 1 to 10. Only reply with a number."]))))


;; a building process loop with a fixed point, it is given an environmental context
;; in every loop we first gather and refine requirements
;; we derive subgoals to currently pursue
;; we define tests and progress metrics
;; then we do an iteration given the system so far
;; and measure the progress
;; we then check if we have reached a fixed point

subgoals (expert expert-role (str "You have this goal: " goal "\nWith these requirements: " requirements "\nAnd these subgoals derived so far:" subgoals "\nUpdate the subgoals if reasonable and enumerate them as a comma separated list: "))
tests (expert expert-role (str "You have this goal: " goal "\nWith these requirements: " requirements "\nAnd these subgoals: " subgoals "\nAnd these tests derived so far: " tests "\n Update the tests if reasonable and return them as Clojure code string that can be passed to eval: "))
progress-metrics (expert expert-role (str "You have this goal: " goal "\nWith these requirements: " requirements "\nAnd these subgoals: " subgoals "\nAnd these tests: " tests "\nAnd these progress metrics derived so far: " progress-metrics "\n Update the progress metrics if reasonable and return them only as a single Clojure code string that can be passed to eval directly: "))
iteration (expert expert-role (str "You have this goal: " goal "\nWith these requirements: " requirements "\nAnd these subgoals: " subgoals "\nAnd these tests: " tests "\nAnd these progress metrics: " progress-metrics "\nAnd this iteration derived so far: " iteration "\n Update the iteration if reasonable and return it as a single Clojure code string that can be passed to eval: "))
progress (expert expert-role (str "You have this goal: " goal "\nWith these requirements: " requirements "\nAnd these subgoals: " subgoals "\nAnd these tests: " tests "\nAnd these progress metrics: " progress-metrics "\nAnd this iteration: " iteration "\nAnd this progress derived so far: " progress "\n Update the progress metric if reasonable and return it as Clojure code. The progress metric function needs to return a real number: "))


(defn build [ctx goal]
(let [expert-role (openai/chat "gpt-4o"
(interleave-with-delimiters
["Given this context:" ctx
"What expert would you hire to pursue this goal:" goal
"Answer a descriptive job title."]))]
(loop [requirements ""
subgoals ""
tests ""
progress-metrics ""
iteration ""
progress "" ]
(let [requirements (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements derived so far:" requirements
"What are the requirements?"]))
subgoals (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements:" requirements
"And these subgoals derived so far:" subgoals
"Update the subgoals if reasonable and enumerate them as a comma separated list: "]))
tests (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements:" requirements
"And these subgoals:" subgoals
"And these tests derived so far:" tests
" Update the tests if reasonable and return them as Clojure code string that can be passed to eval: "]))
progress-metrics (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements:" requirements
"And these subgoals:" subgoals
"And these tests:" tests
"And these progress metrics derived so far:" progress-metrics
" Update the progress metrics if reasonable and return them only as a single Clojure code string that can be passed to eval directly: "]))
iteration (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements:" requirements
"And these subgoals:" subgoals
"And these tests:" tests
"And these progress metrics:" progress-metrics
"And this iteration derived so far:" iteration
" Update the iteration if reasonable and return it as a single Clojure code string that can be passed to eval: "]))
progress (expert expert-role (interleave-with-delimiters ["You have this goal:" goal
"With these requirements:" requirements
"And these subgoals:" subgoals
"And these tests:" tests
"And these progress metrics:" progress-metrics
"And this iteration:" iteration
"And this progress derived so far:" progress
" Update the progress metric if reasonable and return it as Clojure code. The progress metric function needs to return a real number: "]))]
(println "Requirements: " requirements)
(println "Subgoals: " subgoals)
(println "Tests: " tests)
(println "Progress Metrics: " progress-metrics)
(println "Iteration: " iteration)
(println "Progress: " progress)
[expert-role requirements subgoals tests progress-metrics iteration progress]))))


(comment

(def build-result (build "exploring games" "Build a Snake game in Clojure without any external libraries. You have clojure.core, core.async and Datahike available."))

(let [[expert-role requirements subgoals tests progress-metrics iteration progress] build-result]
(println iteration))

)
88 changes: 88 additions & 0 deletions src/ie/simm/processes/thesis.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
(ns ie.simm.processes.thesis
(:require [clojure.core.async :as async :refer [timeout]]
[superv.async :refer [go-try go-loop-try <? S]]
[ie.simm.runtimes.openai :as openai]))

(require '[ie.simm.runtimes.openai :as openai])

(require '[datahike.api :as d])

(openai/chat "gpt-4o" "Hello, I am a chatbot. I am here to help you with your thesis. What do you need help with?")

;; define process and used languages
(comment
(defprocess conversation [chat gen-ai core-async]
(go-try S

))


)

;; objective: have a complete process description that can be simulated

(defn purchase [system & args]
(println "Purchasing" system args)
(last args))

(defn sale [system & args]
(println "Selling" system args))

(def minutes (* 60 1000))

(defn write []
(let [topics ["structure inversion" "structure learning" "structure prediction" "structure analysis"]
topic (rand-nth topics)
writing-time (purchase "ch_weil@topiq.es" ::writing-time ::minutes 25)
topic (purchase "Thesis" ::writing-issue ::issue topic)]
(go-try S
(when topic
(println "Working on topic" topic)
(<? S (timeout (* writing-time minutes)))))))

(defn pause []
(let [recovery-time (purchase "ch_weil@topiq.es" ::recovery-time ::minutes 5)]
(timeout (* recovery-time minutes))))


(defn yesno [question]
(println question)
(go-loop-try S [answer (read-line)]
(if-not ({"yes" "no"} answer)
(do
(println "Please answer yes or no")
(recur (read-line)))
(if (= "yes" answer) true false))))

(defn read-integer [question]
(println question)
(go-loop-try S [answer (read-line)]
(if-not (try (Integer/parseInt answer) (catch Exception e nil))
(do
(println "Please enter an integer")
(recur (read-line)))
answer)))

(comment
(read-integer "How many pages did you write today?")

)

(defn workday []
(go-try S
(purchase "ch_weil@topiq.es" ::simmie-fee ::usd 0.01)
(purchase "Household Toronto Rd." ::coffee ::usd 0.5)
(<? S (go-loop-try S []
(<? S (write))
(<? S (pause))
(when (<? S (yesno "Do you want to continue writing?"))
(recur))))
(sale "PhD program" ::pages ::written (read-integer "How many pages did you write today?"))))

(comment

(read-line)

(workday)

)
8 changes: 4 additions & 4 deletions src/ie/simm/runtimes/assistance.clj
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Properties: stateful, persistent, durable"
(:require [ie.simm.languages.bindings :as lb]
[ie.simm.languages.gen-ai :refer [cheap-llm reasoner-llm stt-basic image-gen]]
[ie.simm.languages.gen-ai :refer [cheap-llm cheap-llm stt-basic image-gen]]
[ie.simm.languages.web-search :refer [search]]
[ie.simm.languages.browser :refer [extract-body]]
[ie.simm.languages.chat :refer [send-text! send-photo! send-document!]]
Expand All @@ -29,7 +29,7 @@
(go-try S
(debug "=========================== SUMMARIZING ===============================")
(let [db @conn
summarization (<? S (reasoner-llm (format pr/summarization conv)))
summarization (<? S (cheap-llm (format pr/summarization conv)))
messages (->> (d/q '[:find ?d ?e
:in $ ?chat
:where
Expand All @@ -47,7 +47,7 @@
(go-for S [[i note] (partition 2 (interleave (iterate inc 2) note-titles))
:let [[nid body] (first (d/q '[:find ?n ?b :in $ ?t :where [?n :note/title ?t] [(get-else $ ?n :note/body "EMPTY") ?b]] db note))
prompt (format pr/note note body #_summarization conv)
new-body (<? S (reasoner-llm prompt))]
new-body (<? S (cheap-llm prompt))]
:when (not (.contains new-body "SKIP"))
:let [new-refs (extract-links new-body)
ref-ids (mapv first (d/q '[:find ?n
Expand Down Expand Up @@ -505,7 +505,7 @@
conv
(str (java.util.Date.)))
_ (debug "prompt" assist-prompt)
reply (<? S (reasoner-llm assist-prompt))
reply (<? S (cheap-llm assist-prompt))
_ (debug "reply" reply)

;; 5. dispatch
Expand Down
4 changes: 2 additions & 2 deletions src/ie/simm/runtimes/etaoin.clj
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
[superv.async :refer [S go-loop-try <? put?]]))

(defn extract-body [url]
(e/with-chrome-headless driver
(e/with-firefox-headless driver
(e/go driver url)
(e/get-element-text driver {:tag :body})))

Expand All @@ -29,4 +29,4 @@
:response (try (extract-body url) (catch Exception e e))))
(recur (<? S get-body))))

[S peer [next-in out]]))
[S peer [next-in out]]))
27 changes: 14 additions & 13 deletions src/ie/simm/runtimes/openai.clj
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@
(def window-sizes {"gpt-3.5-turbo-0125" 16384
"gpt-4-turbo" 128000
"gpt-4o" 128000
"gpt-4o-2024-08-06" 128000
"gpt-4o-mini" 128000 })

(defn chat [model text]
(if (>= (count text) (* 4 (window-sizes model)))
(do
(warn "text too long for " model ": " (count text) (window-sizes model))
(warn "Text too long for " model ": " (count text) (window-sizes model))
(throw (ex-info "Sorry, the text is too long for this model. Please try a shorter text." {:type ::text-too-long :model model :text-start (subs text 0 100) :count (count text)})))
(let [res (create :model model :messages [{:role "system" :content text}])]
(py.- (py.- (first (py.- res choices)) message) content))))
Expand Down Expand Up @@ -55,16 +56,16 @@

(defn openai [[S peer [in out]]]
(let [p (pub in (fn [{:keys [type]}]
(or ({:ie.simm.languages.gen-ai/cheap-llm ::gpt-35-turbo
:ie.simm.languages.gen-ai/reasoner-llm ::gpt-4-turbo
(or ({:ie.simm.languages.gen-ai/cheap-llm ::gpt-4o-mini
:ie.simm.languages.gen-ai/reasoner-llm ::gpt-4o
:ie.simm.languages.gen-ai/stt-basic ::whisper-1
:ie.simm.languages.gen-ai/image-gen ::dall-e-3} type)
:unrelated)))
gpt-35-turbo (chan)
_ (sub p ::gpt-35-turbo gpt-35-turbo)
gpt-4o-mini (chan)
_ (sub p ::gpt-4o-mini gpt-4o-mini)

gpt-4-turbo (chan)
_ (sub p ::gpt-4-turbo gpt-4-turbo)
gpt-4o (chan)
_ (sub p ::gpt-4o gpt-4o)

whisper-1 (chan)
_ (sub p ::whisper-1 whisper-1)
Expand All @@ -76,21 +77,21 @@
_ (sub p :unrelated next-in)]
;; TODO use async http requests for parallelism
;; TODO factor dedicated translator to LLM language
(go-loop-try S [{[m] :args :as s} (<? S gpt-35-turbo)]
(go-loop-try S [{[m] :args :as s} (<? S gpt-4o-mini)]
(when s
(go-try S
(put? S out (assoc s
:type :ie.simm.languages.gen-ai/cheap-llm-reply
:response (try (chat "gpt-3.5-turbo-0125" m) (catch Exception e e)))))
(recur (<? S gpt-35-turbo))))
:response (try (chat "gpt-4o-mini" m) (catch Exception e e)))))
(recur (<? S gpt-4o-mini))))

(go-loop-try S [{[m] :args :as s} (<? S gpt-4-turbo)]
(go-loop-try S [{[m] :args :as s} (<? S gpt-4o)]
(when s
(go-try S
(put? S out (assoc s
:type :ie.simm.languages.gen-ai/reasoner-llm-reply
:response (try (chat "gpt-4-turbo" m) (catch Exception e e)))))
(recur (<? S gpt-4-turbo))))
:response (try (chat "gpt-4o-2024-08-06" m) (catch Exception e e)))))
(recur (<? S gpt-4o))))

(go-loop-try S [{[m] :args :as s} (<? S whisper-1)]
(when s
Expand Down

0 comments on commit c8db31b

Please sign in to comment.