From e2cf3b7aca3c299e8016705cc5baeaae49b9c8e8 Mon Sep 17 00:00:00 2001
From: henk717 <henk@henk.tech>
Date: Sun, 10 Dec 2023 14:30:17 +0100
Subject: [PATCH] koboldcpp.sh - The Mamba Multitool (#554)

* .sh script V1

* koboldcpp.sh polish

* koboldcpp.sh dist generator

* Include html's in dist

* RWKV in Linux Dist

* Lower dependency requirements

* Eliminate wget dependency

* More distinct binary name

I know its technically amd64, but I don't want to cause confusion among nvidia users.

* Use System OpenCL

Unsure how this will behave in the pyinstaller build, but pocl ended up CPU only. With a bit of luck the pyinstaller uses the one from the actual system if compiled in a system without opencl, while conda now includes it for that specific system.

* Add cblas dependency

Missing this causes compile failures on some system's

* ICD workaround

Ideally we find a better solution, but conda forces ICD and needs this for the successful compile. However, pyinstaller then embeds the ICD causing it to be limited to the system it was compiled for. By temporarily removing the ICD pyinstaller can't find it and everything remains functional. Ideally we do this on a pyinstaller level, but I could not find any good options to do so yet.

---------

Co-authored-by: root <root@DESKTOP-DQ1QRAG>
---
 .gitignore       |  3 +++
 environment.yaml | 24 ++++++++++++++++++++++++
 koboldcpp.sh     | 22 ++++++++++++++++++++++
 3 files changed, 49 insertions(+)
 create mode 100644 environment.yaml
 create mode 100755 koboldcpp.sh

diff --git a/.gitignore b/.gitignore
index f89ef9a960154..d84050a5922d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -118,5 +118,8 @@ hipblas.dll
 koboldcpp_hipblas.so
 koboldcpp_hipblas.dll
 
+bin/
+conda/
+
 # Jetbrains idea folder
 .idea/
diff --git a/environment.yaml b/environment.yaml
new file mode 100644
index 0000000000000..17a698466d20c
--- /dev/null
+++ b/environment.yaml
@@ -0,0 +1,24 @@
+name: koboldcpp
+channels:
+  - nvidia/label/cuda-11.5.0
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.8
+  - cuda-nvcc=11.5
+  - cuda-libraries-dev=11.5
+  - cxx-compiler
+  - gxx=10
+  - pip
+  - git=2.35.1
+  - libopenblas
+  - openblas
+  - clblast
+  - ninja
+  - make
+  - packaging
+  - pyinstaller
+  - libcblas
+  - ocl-icd-system
+  - pip:
+    - customtkinter
diff --git a/koboldcpp.sh b/koboldcpp.sh
new file mode 100755
index 0000000000000..50074e402e176
--- /dev/null
+++ b/koboldcpp.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+if [ ! -f "bin/micromamba" ]; then
+	curl -Ls https://anaconda.org/conda-forge/micromamba/1.5.3/download/linux-64/micromamba-1.5.3-0.tar.bz2 | tar -xvj bin/micromamba
+fi
+
+if [[ ! -f "conda/envs/linux/bin/python" || $1 == "rebuild" ]]; then
+	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.yaml -y
+	bin/micromamba create --no-shortcuts -r conda -n linux -f environment.yaml -y
+	bin/micromamba run -r conda -n linux make clean
+fi
+
+bin/micromamba run -r conda -n linux make LLAMA_OPENBLAS=1 LLAMA_CLBLAST=1 LLAMA_CUBLAS=1 LLAMA_PORTABLE=1
+
+if [[ $1 == "rebuild" ]]; then
+	echo Rebuild complete, you can now try to launch Koboldcpp.
+elif [[ $1 == "dist" ]]; then
+	bin/micromamba remove -r conda -n linux --force ocl-icd -y
+	bin/micromamba run -r conda -n linux pyinstaller --noconfirm --onefile --collect-all customtkinter --add-data='./koboldcpp_default.so:.' --add-data='./koboldcpp_cublas.so:.' --add-data='./koboldcpp_openblas.so:.' --add-data='./koboldcpp_clblast.so:.' --add-data='./koboldcpp_clblast_noavx2.so:.' --add-data='./klite.embd:.' --add-data='./kcpp_docs.embd:.' --add-data='./rwkv_vocab.embd:.' --add-data='./rwkv_world_vocab.embd:.' --clean --console koboldcpp.py -n "koboldcpp-linux-x64"
+	bin/micromamba install -r conda -n linux ocl-icd -c conda-forge -y
+ else
+	bin/micromamba run -r conda -n linux python koboldcpp.py $*
+fi