Skip to content

Commit

Permalink
temporary refactor llama_vision_graph_builder
Browse files Browse the repository at this point in the history
  • Loading branch information
ngxson committed Jan 22, 2025
1 parent 32daa38 commit 9716c7b
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 120 deletions.
8 changes: 6 additions & 2 deletions examples/vision/vision.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ static llama_vision_bitmap * load_image_from_file(const char * fname) {
}

// split string by a `std::string delim` instead of `char delim`
static std::vector<std::string> string_split(std::string s, const std::string & delimiter) {
static std::vector<std::string> string_split_str(std::string s, const std::string & delimiter) {
std::vector<std::string> tokens;
size_t pos = 0;
std::string token;
Expand All @@ -76,7 +76,7 @@ static std::vector<tokenized_part> tokenize_with_img_placement(
const std::string & text,
bool add_special,
bool parse_special) {
std::vector<std::string> parts = string_split(text, IMG_PLACEMENT);
std::vector<std::string> parts = string_split_str(text, IMG_PLACEMENT);
std::vector<tokenized_part> output;
for (const auto & part : parts) {
//printf("tokenizing part: %s\n", part.c_str());
Expand Down Expand Up @@ -114,6 +114,10 @@ int main(int argc, char ** argv) {
llama_context * ctx = llama_init.context.get();
const llama_model * model = llama_init.model.get();
const llama_vocab * vocab = llama_model_get_vocab(model);
if (!model) {
LOG_ERR("failed to load model\n");
return 1;
}

struct common_sampler * smpl = common_sampler_init(model, params.sampling);

Expand Down
5 changes: 5 additions & 0 deletions src/llama-model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4056,6 +4056,11 @@ enum llama_rope_type llama_model_rope_type(const struct llama_model * model) {
case LLM_ARCH_QWEN2VL:
return LLAMA_ROPE_TYPE_MROPE;

case LLM_ARCH_VISION_LLAVA:
case LLM_ARCH_VISION_MOBILEVLM:
case LLM_ARCH_VISION_MINICPMV:
GGML_ABORT("vision arch does not use RoPE");

// all model arches should be listed explicitly here
case LLM_ARCH_UNKNOWN:
GGML_ABORT("unknown architecture");
Expand Down
Loading

0 comments on commit 9716c7b

Please sign in to comment.