example-gemma2-9B-27B.env

MODEL_PATH="/absolute/path/to/model/gemma2-it.gguf"

BOT=""
PREFIX_TEMPLATE="<start_of_turn>{agent}
"
EOS="<end_of_turn>
"
AGENT_USER="user"
AGENT_ASSISTANT="model"
ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"

REAL_TIME=1
N_CTX=8000
N_GENERATE=2000
TEMPERATURE=0.8
TOP_P=0.9
TOP_K=40
SEED=69
USE_MMAP=1
USE_MLOCK=1
USE_GPU=1