From 868b63ff776ebeebcaf51f7129cb4ac3fa72430c Mon Sep 17 00:00:00 2001 From: jmorganca Date: Sat, 18 Apr 2026 23:14:38 -0700 Subject: [PATCH] llama/compat: load Ollama-format GGUFs in llama-server Squashed from upstream/jmorganca/llama-compat on 2026-04-29. Source tip: 0c33775d378511a9b3c7f2e3b80eda355511d9f3. Original source commits: - 25223160d llama/compat: add in-memory shim so llama-server can load Ollama-format GGUFs - 7449b539a llm,server: route Ollama-format gemma3 blobs through llama/compat - 436f2e2b1 llama/compat: make patch-apply idempotent - 8c2c9d4c8 llama/compat: extend gemma3 handler to cover 1B and 270M blobs - 021389f7b llama/compat: shrink clip.cpp injection from 18 lines to 1 - 61b367ec2 llama/compat: shrink patch to pure call-site hooks (34 -> 20 lines) - 36049361c llama/compat: simplify shim (gemma3-tested) - 8fa664865 llama/compat: add qwen35moe text handler - db0c74530 llama/compat: add qwen35moe vision (clip) support - 2a388da77 llama/compat: split shared infra into a util TU - 9a69a17dc llama/compat: document non-public API dependencies - d0f38a915 llama/compat: add gpt-oss and lfm2 handlers - 086071822 llama/compat: add mistral3 text handler (vision TODO) - 63bde9ff7 llama/compat: add mistral3 vision (clip) support - 3a57b89d5 llama/compat: apply LLaMA RoPE permute to mistral3 vision Q/K - 99cb87439 llama/compat: add qwen35, gemma4, deepseek-ocr handlers - 2c7850dba llama/compat: add nemotron_h_moe handler (latent FFN + MTP skip) - 9e3b54225 llama/compat: add llama4 text + clip handlers - 034fee349 llama/compat: add gemma4 clip handler (gemma4v projector) - 9945c5a93 server: remove dhiltgen/* compat redirect table - 5d4539101 llama/compat: rewrite gemma4 tokenizer model to BPE - 7e0765327 llama/compat: add glm-ocr text handler + text-loader load-op hook - f1bd1a25a llama/compat: add glm-ocr clip handler (glm4v projector) - 4b5cf3420 llama/compat: collapse text-loader hook back to one new patch line - eb4ecf4fc llama/compat: extend gemma4 clip handler to gemma4a (audio) - a23a5e76f llama/compat: fix gemma4a per-block norm tensor mapping - cd2dcaff4 llama/compat: add embeddinggemma handler - 1ce8a6b26 llama/compat: add qwen3-vl + qwen2.5-vl handlers - fd98ffa1e llama/compat: add gemma3n + glm4moelite handlers - cc7bdf0bc llama/compat: handle null buft in maybe_load_tensor - 0c33775d3 llama/compat: disable mmap when load_op transforms text-side tensors --- llama/compat/README.md | 121 ++ llama/compat/apply-patch.cmake | 44 + llama/compat/compat.cmake | 56 + llama/compat/llama-ollama-compat-util.cpp | 378 ++++ llama/compat/llama-ollama-compat-util.h | 139 ++ llama/compat/llama-ollama-compat.cpp | 2017 +++++++++++++++++++++ llama/compat/llama-ollama-compat.h | 82 + llama/compat/upstream-edits.patch | 78 + llama/server/CMakeLists.txt | 32 + llm/llama_server.go | 27 + server/model_resolver.go | 63 - server/routes.go | 8 +- 12 files changed, 2975 insertions(+), 70 deletions(-) create mode 100644 llama/compat/README.md create mode 100644 llama/compat/apply-patch.cmake create mode 100644 llama/compat/compat.cmake create mode 100644 llama/compat/llama-ollama-compat-util.cpp create mode 100644 llama/compat/llama-ollama-compat-util.h create mode 100644 llama/compat/llama-ollama-compat.cpp create mode 100644 llama/compat/llama-ollama-compat.h create mode 100644 llama/compat/upstream-edits.patch diff --git a/llama/compat/README.md b/llama/compat/README.md new file mode 100644 index 000000000..193002bac --- /dev/null +++ b/llama/compat/README.md @@ -0,0 +1,121 @@ +# llama.cpp compatibility shim + +This directory holds an in-process compatibility layer that lets upstream +`llama-server` load GGUFs produced by older versions of Ollama (and files +pulled from the Ollama registry) without re-converting or re-downloading. + +The layer is applied automatically at build time via CMake `FetchContent`'s +`PATCH_COMMAND` — there is no separate "apply patches" step. + +## Files + +- `llama-ollama-compat.h`, `llama-ollama-compat.cpp` — the shim itself. These + are regular source files owned by Ollama; they get copied into the fetched + llama.cpp source tree during configure. +- `upstream-edits.patch` — small additive edits to upstream files so the + shim gets called. Currently ~48 lines touching 6 files. Kept as a real + `git` patch so re-generation on upstream bumps is one command. + +## What the shim does + +The shim runs at two well-defined points in the loader: + +1. **After `gguf_init_from_file`**, for both the main model loader and the + `mtmd/clip` loader: inspects the just-parsed metadata and decides whether + the file is an Ollama-format GGUF. If so, it mutates the in-memory + `gguf_context` and `ggml_context` (KV names, tensor names, tensor types) + so the rest of the loader sees an upstream-shape file. + +2. **After `load_all_data`**: applies any numerical fix-ups that need the + tensors in their final backend buffers (e.g. RMSNorm `+1` if a future + arch needs it — gemma3 doesn't). + +Non-Ollama files are detected by the absence of Ollama-specific KV keys +(e.g. `gemma3.mm.tokens_per_image`) or embedded `v.*` / `mm.*` tensors in +the main model file. When no markers are present every compat function is +an immediate no-op. + +## Currently supported architectures + +| Arch | Text loader | Clip (mmproj) loader | +|---|---|---| +| `gemma3` | KV injection (`layer_norm_rms_epsilon`, `rope.freq_base`, `rope.freq_base_swa`), tokenizer vocab truncation, drop `v.*`/`mm.*` tensors | Arch rewrite to `clip`, KV synthesis (`clip.vision.*`, `clip.projector_type=gemma3`), tensor renames (`v.patch_embedding`→`v.patch_embd`, `mlp.fc{1,2}`→`ffn_{down,up}`, etc.), F16→F32 promotion for patch/position embeddings (Metal IM2COL requirement) | +| `qwen35moe` | head_count_kv array → scalar, rope dimension_sections pad 3→4, `ssm_dt`→`ssm_dt.bias` rename, drop `v.*`/`mm.*`/`mtp.*` tensors | Arch rewrite to `clip`, KV synthesis (`clip.vision.*`, `clip.projector_type=qwen3vl_merger`), per-block QKV merge (concat at load time), patch_embed reshape + F16→F32 + slice-as-temporal-pair (reclaiming an orphan `v.blk.0.attn_k` slot for the second pair) | +| `gptoss` | Arch rename `gptoss`→`gpt-oss` (incl. KV prefix), inject `gpt-oss.expert_feed_forward_length` from `ffn_gate_exps` shape, tensor renames (`attn_out`→`attn_output`, `attn_sinks`→`attn_sinks.weight`, `ffn_norm`→`post_attention_norm`) | n/a | +| `lfm2` | Tensor rename `output_norm.weight`→`token_embd_norm.weight`, fix stale `lfm2.feed_forward_length` from `ffn_gate` shape | n/a | +| `mistral3` | RoPE YaRN renames (`rope.scaling.beta_*`→`rope.scaling.yarn_beta_*`), `rope.scaling_beta`→`attention.temperature_scale`, drop `v.*`/`mm.*` tensors | Arch rewrite to `clip`, KV synthesis (`clip.vision.*`, `clip.projector_type=pixtral`), tensor renames (`v.patch_conv`→`v.patch_embd`, `v.encoder_norm`→`v.pre_ln`, `attn_output`→`attn_out`, `attn_norm`/`ffn_norm`→`ln1`/`ln2`, `mm.linear_{1,2}`→`mm.{1,2}`, `mm.norm`→`mm.input_norm`, `mm.patch_merger.merging_layer`→`mm.patch_merger`), zero-fill `v.token_embd.img_break` (reclaims `output_norm.weight` slot — Ollama's monolithic blob doesn't ship this tensor and per-row dequant of token_embd Q4_K is heavyweight; zero-fill makes [IMG_BREAK] insertion a no-op), F32 promote of `v.patch_embd.weight` (Metal IM2COL), LLaMA-style RoPE permute on vision Q/K (Ollama's converter skips repacking `v.*` tensors but pixtral expects HF-permuted layout) | +| `qwen35` | Same fixes as `qwen35moe` (head_count_kv array→scalar, rope dimension_sections pad 3→4, `ssm_dt`→`ssm_dt.bias`, drop `v.*`/`mm.*`/`mtp.*`) but for the non-MoE qwen3.5 (e.g. 9B). Both arches share `apply_qwen35_text_fixes`. | n/a | +| `gemma4` | Drop `a.*`/`v.*`/`mm.*` (audio + vision + projector) from the text loader. Covers both E2B/E4B (dense) and 26B-A4B (MoE). | n/a | +| `deepseekocr` | Arch rename `deepseekocr`→`deepseek2-ocr` (incl. KV prefix), inject `expert_feed_forward_length` from `ffn_down_exps` shape, `expert_shared_count` from `ffn_down_shexp` shape, default `attention.layer_norm_rms_epsilon`, drop `s.*`/`v.*`/`mm.*` | Arch rewrite to `clip`, KV synthesis (`clip.vision.*`, `clip.vision.sam.*`, `clip.projector_type=deepseekocr`, defaults for `feed_forward_length`/`projection_dim`/`window_size`/image stats), prefix-only rename `s.*`→`v.sam.*` (substring rename would corrupt `mm.layers`), CLIP leaf renames (`self_attn.{out,qkv}_proj`→`attn_{out,qkv}`, `layer_norm{1,2}`→`ln{1,2}`, `mlp.fc{1,2}`→`ffn_{up,down}`, `pre_layrnorm`→`pre_ln`), SAM leaf renames (`attn.proj`→`attn.out`, `attn.rel_pos_{h,w}`→`attn.pos_{h,w}.weight`, `norm{1,2}`→`{pre,post}_ln`), projector renames (`mm.layers`→`mm.model.fc`, `mm.image_newline`/`view_seperator`→`v.*`), F32 promote of `v.patch_embd.weight`, `v.sam.patch_embd.weight`, `v.position_embd.weight` | +| `nemotron_h_moe` | For latent-FFN variants (e.g. nemotron-3-super 120B-A12B): inject `moe_latent_size` from `ffn_latent_in.weight` ne[1], rename `ffn_latent_{in,out}`→`ffn_latent_{down,up}`. For all variants: drop `mtp.*` (Multi-Token Prediction tensors that Ollama emits as one-tensor-per-expert; ~1040 extras on the 120B). Standard variants (e.g. nemotron-cascade-2 30B-A3B) load with no rename, only the MTP skip. | n/a | + +Usage: + +``` +llama-server --model /path/to/ollama-blob --mmproj /path/to/ollama-blob +``` + +Passing the same monolithic GGUF as both `--model` and `--mmproj` works — +each loader applies its own translation. + +Additional architectures are added by implementing a `handle_()` +and (for vision models) `handle__clip()` in `llama-ollama-compat.cpp` +and dispatching them from `translate_metadata` / `translate_clip_metadata`. + +## Regenerating `upstream-edits.patch` + +After upstream changes the insertion points (rare), re-apply the edits to +a fresh checkout and run: + +``` +cd /path/to/llama.cpp +git diff -- \ + ggml/include/gguf.h \ + ggml/src/gguf.cpp \ + src/CMakeLists.txt \ + src/llama-model-loader.cpp \ + src/llama-model.cpp \ + tools/mtmd/clip.cpp \ + > /path/to/ollama/llama/compat/upstream-edits.patch +``` + +## Why not fork llama.cpp or vendor it? + +Forking means tracking upstream manually. Vendoring means snapshotting all of +llama.cpp's source in the Ollama tree (the old `llama/llama.cpp/` layout). +This shim keeps upstream unmodified on disk and the Ollama-specific logic +isolated in two files plus a small diff — upstream bumps are usually just +`LLAMA_CPP_VERSION` changes. + +## Maintenance: non-public API dependencies + +The compat code is mostly written against stable public APIs (`gguf.h`, +`ggml.h`, `ggml-backend.h`). There are three places where we lean on +something that isn't strictly public: + +| Hack | Why | Escape hatch if upstream changes | +|---|---|---| +| Direct writes to `ggml_tensor::type` / `ne[]` / `nb[]` | No sanctioned mutator exists for post-creation tensor reshape/retype. Struct is public so this works today. | Ask upstream to expose `ggml_tensor_set_{type,shape}` helpers, or introduce them in our compat util and submit a PR. | +| `const_cast(gguf_get_tensor_name(...))` in `rename_tensor` | Pointer aims into a mutable `char[GGML_MAX_NAME]` buffer inside a `std::vector` element; the const is API hygiene. Lets us rename gguf tensors without a new public helper. | Add `gguf_rename_tensor` to `gguf.h` (10 lines) and drop the `const_cast`. | +| `llama_model_loader` forward-decl from `src/llama-model-loader.h` | Used only as an opaque pointer key for our skip-prefix registry. Never dereferenced. | Replace with `const void *` in our registry signatures. Zero behavioral change. | + +None of these have changed in years. If an upstream bump breaks any of +them, each has a trivial workaround. See the top of +`llama-ollama-compat-util.h` for the inline notes. + +## Documented hacks inside per-arch handlers + +- **`reclaim_slot_as` (qwen35moe patch_embed split)** — repurposes an + orphaned `v.blk.0.attn_k` slot (left over after the QKV merge) as a + newly-synthesized `v.patch_embd.weight.1`. Needed because clip.cpp's + `ctx_meta` is sized for exactly the original tensor count (no_alloc + branch of `gguf_init_from_file` uses `n_tensors * ggml_tensor_overhead()` + with zero slack). Comment in the helper and call site explains the + reasoning; replacement would be a 1-line upstream patch that adds small + slack to the ctx size. + +- **Load-op registry overrides `file_offset`** — `maybe_load_tensor` gets + passed the gguf offset by its caller but ignores it when a registered + op exists. Intentional: the ops capture their own source offsets at + translate time (before our renames invalidate them). Documented in the + op-registration helpers. diff --git a/llama/compat/apply-patch.cmake b/llama/compat/apply-patch.cmake new file mode 100644 index 000000000..708be4332 --- /dev/null +++ b/llama/compat/apply-patch.cmake @@ -0,0 +1,44 @@ +# Idempotent patch applier used by compat.cmake. +# +# Invocation (from a CMake PATCH_COMMAND): +# cmake -DPATCH_FILE= -P apply-patch.cmake +# +# The patch is applied in the current working directory (which ExternalProject +# / FetchContent sets to the fetched source's SOURCE_DIR). If the patch is +# already applied — detected via `git apply --reverse --check` — this script +# is a no-op. This makes re-configuring and re-building the project safe. + +if(NOT DEFINED PATCH_FILE) + message(FATAL_ERROR "apply-patch.cmake: PATCH_FILE not set") +endif() +if(NOT EXISTS "${PATCH_FILE}") + message(FATAL_ERROR "apply-patch.cmake: PATCH_FILE does not exist: ${PATCH_FILE}") +endif() + +find_package(Git QUIET REQUIRED) + +# If the patch can be REVERSED cleanly, it's already applied. Skip. +execute_process( + COMMAND ${GIT_EXECUTABLE} apply --reverse --check "${PATCH_FILE}" + RESULT_VARIABLE _reverse_check + OUTPUT_QUIET ERROR_QUIET +) +if(_reverse_check EQUAL 0) + message(STATUS "llama/compat: patch already applied, skipping") + return() +endif() + +# Otherwise, apply forward. +execute_process( + COMMAND ${GIT_EXECUTABLE} apply --whitespace=nowarn "${PATCH_FILE}" + RESULT_VARIABLE _apply_result +) +if(NOT _apply_result EQUAL 0) + message(FATAL_ERROR + "llama/compat: failed to apply ${PATCH_FILE}\n" + "This usually means upstream llama.cpp has drifted. " + "Regenerate the patch (see llama/compat/README.md) against the " + "pinned LLAMA_CPP_VERSION and retry.") +endif() + +message(STATUS "llama/compat: applied patch") diff --git a/llama/compat/compat.cmake b/llama/compat/compat.cmake new file mode 100644 index 000000000..9c8406d8e --- /dev/null +++ b/llama/compat/compat.cmake @@ -0,0 +1,56 @@ +# llama.cpp compatibility shim — CMake integration +# +# Include this file BEFORE calling FetchContent_Declare(llama_cpp ...) to +# patch the fetched upstream llama.cpp with Ollama's in-process compat +# layer. Example usage: +# +# include(${CMAKE_CURRENT_SOURCE_DIR}/../compat/compat.cmake) +# +# FetchContent_Declare( +# llama_cpp +# GIT_REPOSITORY ... +# GIT_TAG ${LLAMA_CPP_GIT_TAG} +# GIT_SHALLOW TRUE +# PATCH_COMMAND ${OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND} +# UPDATE_DISCONNECTED TRUE +# ) +# +# The compat layer consists of: +# 1. Two new source files dropped into the fetched tree's src/ +# (llama-ollama-compat.{h,cpp}) — Ollama-owned. +# 2. A small patch (upstream-edits.patch) that wires the new files into +# the build and adds call-sites in upstream loaders. + +set(_compat_dir ${CMAKE_CURRENT_LIST_DIR}) + +# Expose a single variable the main CMakeLists passes into FetchContent's +# PATCH_COMMAND. The patch is applied via a small CMake script so the step +# is idempotent — re-configuring or rebuilding won't fail with "already +# applied". +# +# The compat source files (.h, .cpp) are NOT copied into the fetched tree. +# Instead, llama/server/CMakeLists.txt does target_sources() on the llama +# target after FetchContent_MakeAvailable. That keeps Ollama's code in +# Ollama's tree and makes the patch pure call-site insertions. +set(OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND + ${CMAKE_COMMAND} + -DPATCH_FILE=${_compat_dir}/upstream-edits.patch + -P ${_compat_dir}/apply-patch.cmake + CACHE INTERNAL "llama.cpp compat patch command for FetchContent") + +# Where the compat source files live, so the main CMakeLists can wire them +# into the llama target. +set(OLLAMA_LLAMA_CPP_COMPAT_DIR + "${_compat_dir}" + CACHE INTERNAL "Directory holding llama-ollama-compat.{h,cpp}") + +# Also export the individual paths in case callers want to do something +# custom (e.g. emit a dependency on the patch so reconfigures re-apply). +set(OLLAMA_LLAMA_CPP_COMPAT_PATCH_FILE + "${_compat_dir}/upstream-edits.patch" + CACHE INTERNAL "Path to the llama.cpp compat patch") + +set(OLLAMA_LLAMA_CPP_COMPAT_SOURCES + "${_compat_dir}/llama-ollama-compat.h" + "${_compat_dir}/llama-ollama-compat.cpp" + CACHE INTERNAL "Source files copied into llama.cpp's src/ dir") diff --git a/llama/compat/llama-ollama-compat-util.cpp b/llama/compat/llama-ollama-compat-util.cpp new file mode 100644 index 000000000..ce5d38bb3 --- /dev/null +++ b/llama/compat/llama-ollama-compat-util.cpp @@ -0,0 +1,378 @@ +#include "llama-ollama-compat-util.h" + +#include "llama-impl.h" +#include "llama-model-loader.h" + +#include +#include +#include +#include +#include + +namespace llama_ollama_compat::detail { + +// ------------------------------------------------------------------------- +// gguf_context KV helpers +// ------------------------------------------------------------------------- + +bool has_key(const gguf_context * meta, const char * key) { + return gguf_find_key(meta, key) >= 0; +} + +void copy_u32_kv(gguf_context * meta, const char * src, const char * dst) { + if (has_key(meta, dst)) return; + const int64_t k = gguf_find_key(meta, src); + if (k < 0) return; + gguf_set_val_u32(meta, dst, gguf_get_val_u32(meta, k)); +} + +void copy_f32_kv(gguf_context * meta, const char * src, const char * dst) { + if (has_key(meta, dst)) return; + const int64_t k = gguf_find_key(meta, src); + if (k < 0) return; + gguf_set_val_f32(meta, dst, gguf_get_val_f32(meta, k)); +} + +void copy_kv(gguf_context * meta, const char * src, const char * dst) { + if (has_key(meta, dst)) return; + const int64_t kid = gguf_find_key(meta, src); + if (kid < 0) return; + const enum gguf_type t = gguf_get_kv_type(meta, kid); + switch (t) { + case GGUF_TYPE_UINT8: gguf_set_val_u8 (meta, dst, gguf_get_val_u8 (meta, kid)); break; + case GGUF_TYPE_INT8: gguf_set_val_i8 (meta, dst, gguf_get_val_i8 (meta, kid)); break; + case GGUF_TYPE_UINT16: gguf_set_val_u16 (meta, dst, gguf_get_val_u16 (meta, kid)); break; + case GGUF_TYPE_INT16: gguf_set_val_i16 (meta, dst, gguf_get_val_i16 (meta, kid)); break; + case GGUF_TYPE_UINT32: gguf_set_val_u32 (meta, dst, gguf_get_val_u32 (meta, kid)); break; + case GGUF_TYPE_INT32: gguf_set_val_i32 (meta, dst, gguf_get_val_i32 (meta, kid)); break; + case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (meta, dst, gguf_get_val_f32 (meta, kid)); break; + case GGUF_TYPE_BOOL: gguf_set_val_bool(meta, dst, gguf_get_val_bool(meta, kid)); break; + case GGUF_TYPE_STRING: gguf_set_val_str (meta, dst, gguf_get_val_str (meta, kid)); break; + case GGUF_TYPE_UINT64: gguf_set_val_u64 (meta, dst, gguf_get_val_u64 (meta, kid)); break; + case GGUF_TYPE_INT64: gguf_set_val_i64 (meta, dst, gguf_get_val_i64 (meta, kid)); break; + case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (meta, dst, gguf_get_val_f64 (meta, kid)); break; + case GGUF_TYPE_ARRAY: { + const enum gguf_type et = gguf_get_arr_type(meta, kid); + const size_t n = gguf_get_arr_n(meta, kid); + if (et == GGUF_TYPE_STRING) { + std::vector owned; + owned.reserve(n); + std::vector ptrs; + ptrs.reserve(n); + for (size_t i = 0; i < n; ++i) owned.emplace_back(gguf_get_arr_str(meta, kid, i)); + for (const auto & s : owned) ptrs.push_back(s.c_str()); + gguf_set_arr_str(meta, dst, ptrs.data(), n); + } else { + gguf_set_arr_data(meta, dst, et, gguf_get_arr_data(meta, kid), n); + } + break; + } + default: break; + } +} + +void rename_kv_prefix(gguf_context * meta, const char * old_prefix, + const char * new_prefix) { + const size_t old_len = std::strlen(old_prefix); + // Snapshot keys first; copy_kv() invalidates the kv index by appending. + std::vector matches; + const int64_t n = gguf_get_n_kv(meta); + for (int64_t i = 0; i < n; ++i) { + const char * k = gguf_get_key(meta, i); + if (std::strncmp(k, old_prefix, old_len) == 0) matches.emplace_back(k); + } + for (const auto & old_key : matches) { + copy_kv(meta, old_key.c_str(), + (std::string(new_prefix) + old_key.substr(old_len)).c_str()); + } +} + +void inject_u32_if_missing (gguf_context * meta, const char * key, uint32_t v) { + if (!has_key(meta, key)) gguf_set_val_u32(meta, key, v); +} +void inject_f32_if_missing (gguf_context * meta, const char * key, float v) { + if (!has_key(meta, key)) gguf_set_val_f32(meta, key, v); +} +void inject_str_if_missing (gguf_context * meta, const char * key, const char * v) { + if (!has_key(meta, key)) gguf_set_val_str(meta, key, v); +} +void inject_bool_if_missing(gguf_context * meta, const char * key, bool v) { + if (!has_key(meta, key)) gguf_set_val_bool(meta, key, v); +} +void inject_f32_arr_if_missing(gguf_context * meta, const char * key, + const float * data, size_t n) { + if (!has_key(meta, key)) gguf_set_arr_data(meta, key, GGUF_TYPE_FLOAT32, data, n); +} + +void truncate_str_arr(gguf_context * meta, const char * key, size_t new_n) { + const int64_t kid = gguf_find_key(meta, key); + if (kid < 0 || new_n >= gguf_get_arr_n(meta, kid)) return; + + std::vector owned; + owned.reserve(new_n); + std::vector ptrs; + ptrs.reserve(new_n); + for (size_t i = 0; i < new_n; ++i) owned.emplace_back(gguf_get_arr_str(meta, kid, i)); + for (const auto & s : owned) ptrs.push_back(s.c_str()); + gguf_set_arr_str(meta, key, ptrs.data(), new_n); +} + +void truncate_data_arr(gguf_context * meta, const char * key, + gguf_type elem_type, size_t elem_size, size_t new_n) { + const int64_t kid = gguf_find_key(meta, key); + if (kid < 0 || new_n >= gguf_get_arr_n(meta, kid)) return; + + std::vector copy(elem_size * new_n); + std::memcpy(copy.data(), gguf_get_arr_data(meta, kid), elem_size * new_n); + gguf_set_arr_data(meta, key, elem_type, copy.data(), new_n); +} + +// ------------------------------------------------------------------------- +// ggml_context tensor scans +// ------------------------------------------------------------------------- + +bool any_tensor_with_prefix(const ggml_context * ctx, const char * prefix) { + const size_t plen = std::strlen(prefix); + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) { + if (std::strncmp(ggml_get_name(t), prefix, plen) == 0) return true; + } + return false; +} + +// ------------------------------------------------------------------------- +// Tensor renaming / reshaping (mutates both contexts) +// ------------------------------------------------------------------------- + +// gguf_get_tensor_name returns a pointer into a mutable `char[GGML_MAX_NAME]` +// inside a std::vector element; the const on the return type is API +// courtesy, so writing through const_cast is defined. +void rename_tensor(gguf_context * meta, ggml_context * ctx, + const char * old_name, const char * new_name) { + const int64_t id = gguf_find_tensor(meta, old_name); + if (id < 0) return; + if (char * p = const_cast(gguf_get_tensor_name(meta, id))) { + std::strncpy(p, new_name, GGML_MAX_NAME - 1); + p[GGML_MAX_NAME - 1] = '\0'; + } + if (ggml_tensor * t = ggml_get_tensor(ctx, old_name)) ggml_set_name(t, new_name); +} + +void rename_tensors_containing(gguf_context * meta, ggml_context * ctx, + const char * needle, const char * replacement) { + std::vector> renames; + const int64_t n = gguf_get_n_tensors(meta); + const size_t needle_len = std::strlen(needle); + for (int64_t i = 0; i < n; ++i) { + std::string s(gguf_get_tensor_name(meta, i)); + const size_t pos = s.find(needle); + if (pos == std::string::npos) continue; + std::string ns = s; + ns.replace(pos, needle_len, replacement); + renames.emplace_back(std::move(s), std::move(ns)); + } + for (const auto & [from, to] : renames) rename_tensor(meta, ctx, from.c_str(), to.c_str()); +} + +void set_tensor_type(ggml_tensor * t, ggml_type type) { + t->type = type; + t->nb[0] = ggml_type_size(type); + t->nb[1] = t->nb[0] * (t->ne[0] / ggml_blck_size(type)); + for (int i = 2; i < GGML_MAX_DIMS; ++i) t->nb[i] = t->nb[i - 1] * t->ne[i - 1]; +} + +void set_tensor_shape(ggml_tensor * t, std::initializer_list shape) { + int i = 0; + for (auto v : shape) t->ne[i++] = v; + for (; i < GGML_MAX_DIMS; ++i) t->ne[i] = 1; + set_tensor_type(t, t->type); +} + +// Rename an orphan tensor slot as a new synthesized tensor. See header for +// why this is the workaround of choice (clip's ctx_meta has no spare capacity). +bool reclaim_slot_as(gguf_context * meta, ggml_context * ctx, + const char * orphan_name, const char * new_name, + std::initializer_list shape, ggml_type type) { + if (gguf_find_tensor(meta, orphan_name) < 0) return false; + rename_tensor(meta, ctx, orphan_name, new_name); + ggml_tensor * t = ggml_get_tensor(ctx, new_name); + if (!t) return false; + set_tensor_shape(t, shape); + set_tensor_type (t, type); + return true; +} + +size_t tensor_file_offset(const gguf_context * meta, const char * name) { + const int64_t id = gguf_find_tensor(meta, name); + if (id < 0) return 0; + return gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, id); +} + +// ------------------------------------------------------------------------- +// Per-loader skip-prefix registry +// ------------------------------------------------------------------------- + +namespace { +std::mutex g_skip_mutex; +std::unordered_map> g_skip_prefixes; +} // anon + +void add_skip_prefix(const llama_model_loader * ml, std::string prefix) { + std::lock_guard lk(g_skip_mutex); + g_skip_prefixes[ml].push_back(std::move(prefix)); +} + +bool should_skip_tensor_prefix(const llama_model_loader * ml, const char * name) { + std::lock_guard lk(g_skip_mutex); + auto it = g_skip_prefixes.find(ml); + if (it == g_skip_prefixes.end()) return false; + for (const auto & prefix : it->second) { + if (std::strncmp(name, prefix.c_str(), prefix.size()) == 0) return true; + } + return false; +} + +namespace { +std::mutex g_no_mmap_mutex; +std::unordered_set g_no_mmap; +} // anon + +void disable_mmap_for(const llama_model_loader * ml) { + std::lock_guard lk(g_no_mmap_mutex); + g_no_mmap.insert(ml); +} + +bool is_mmap_disabled_for(const llama_model_loader * ml) { + std::lock_guard lk(g_no_mmap_mutex); + return g_no_mmap.count(ml) > 0; +} + +// ------------------------------------------------------------------------- +// Load-time transform registry +// ------------------------------------------------------------------------- + +namespace { +std::mutex g_loadop_mutex; +std::unordered_map g_loadops; +} // anon + +void register_load_op(std::string dest_name, LoadOp op) { + std::lock_guard lk(g_loadop_mutex); + g_loadops[std::move(dest_name)] = std::move(op); +} + +bool take_load_op(const char * dest_name, LoadOp & out) { + std::lock_guard lk(g_loadop_mutex); + auto it = g_loadops.find(dest_name); + if (it == g_loadops.end()) return false; + out = std::move(it->second); + g_loadops.erase(it); + return true; +} + +bool read_at(const char * path, size_t offset, void * dst, size_t size) { + FILE * f = std::fopen(path, "rb"); + if (!f) return false; + bool ok = (std::fseek(f, (long) offset, SEEK_SET) == 0 + && std::fread(dst, 1, size, f) == size); + std::fclose(f); + return ok; +} + +// ------------------------------------------------------------------------- +// Common high-level transforms +// ------------------------------------------------------------------------- + +void promote_tensor_to_f32(gguf_context * meta, ggml_context * ctx, const char * name) { + const int64_t tid = gguf_find_tensor(meta, name); + if (tid < 0) return; + ggml_tensor * t = ggml_get_tensor(ctx, name); + if (!t || t->type != GGML_TYPE_F16) return; + + const size_t src_offset = tensor_file_offset(meta, name); + const size_t n_elem = ggml_nelements(t); + const size_t src_size = n_elem * sizeof(uint16_t); + + set_tensor_type(t, GGML_TYPE_F32); + + register_load_op(name, LoadOp{ + [src_offset, src_size, n_elem](const char * path, void * dst, size_t dst_size) { + (void) dst_size; + std::vector src(src_size); + if (!read_at(path, src_offset, src.data(), src_size)) return false; + const uint16_t * sp = reinterpret_cast(src.data()); + float * dp = reinterpret_cast(dst); + for (size_t i = 0; i < n_elem; ++i) dp[i] = ggml_fp16_to_fp32(sp[i]); + return true; + }, + "F16->F32 promote", + }); +} + +void register_concat_load(const gguf_context * meta, std::string dest_name, + const std::vector & src_names) { + std::vector> regions; + regions.reserve(src_names.size()); + for (const auto & n : src_names) { + const int64_t id = gguf_find_tensor(meta, n.c_str()); + if (id < 0) return; + regions.emplace_back( + gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, id), + gguf_get_tensor_size(meta, id)); + } + register_load_op(std::move(dest_name), LoadOp{ + [regions](const char * path, void * dst, size_t dst_size) { + size_t total = 0; + for (auto & [_, sz] : regions) total += sz; + if (total != dst_size) return false; + uint8_t * p = static_cast(dst); + for (auto & [off, sz] : regions) { + if (!read_at(path, off, p, sz)) return false; + p += sz; + } + return true; + }, + "concat sources", + }); +} + +void register_concat_load_to_f32(const gguf_context * meta, + const ggml_context * ctx, + std::string dest_name, + const std::vector & src_names) { + struct Region { size_t offset; size_t size; ggml_type type; size_t n_elem; }; + std::vector regions; + regions.reserve(src_names.size()); + for (const auto & n : src_names) { + const int64_t id = gguf_find_tensor(meta, n.c_str()); + if (id < 0) return; + const ggml_tensor * t = ggml_get_tensor(const_cast(ctx), n.c_str()); + if (!t) return; + regions.push_back({ + gguf_get_data_offset(meta) + gguf_get_tensor_offset(meta, id), + gguf_get_tensor_size(meta, id), + t->type, + (size_t) ggml_nelements(t), + }); + } + register_load_op(std::move(dest_name), LoadOp{ + [regions](const char * path, void * dst, size_t dst_size) { + size_t total_elems = 0; + for (auto & r : regions) total_elems += r.n_elem; + if (total_elems * sizeof(float) != dst_size) return false; + + float * dp = static_cast(dst); + for (auto & r : regions) { + std::vector src(r.size); + if (!read_at(path, r.offset, src.data(), r.size)) return false; + const auto * tt = ggml_get_type_traits(r.type); + if (!tt || !tt->to_float) return false; + tt->to_float(src.data(), dp, (int64_t) r.n_elem); + dp += r.n_elem; + } + return true; + }, + "concat sources (mixed types -> F32)", + }); +} + +} // namespace llama_ollama_compat::detail diff --git a/llama/compat/llama-ollama-compat-util.h b/llama/compat/llama-ollama-compat-util.h new file mode 100644 index 000000000..304ac0393 --- /dev/null +++ b/llama/compat/llama-ollama-compat-util.h @@ -0,0 +1,139 @@ +#pragma once + +// Internal helpers shared by the per-architecture handlers in +// llama-ollama-compat.cpp. Not part of the public API. +// +// Everything lives under namespace llama_ollama_compat::detail. The +// definitions live in llama-ollama-compat-util.cpp, which also owns the +// registry globals (tensor skip list, load-op table) that need a single +// translation unit. +// +// ---- Non-public API dependencies (see also README.md "Maintenance") ---- +// +// Mostly public: gguf_* and ggml_* accessors from ggml/include/ are all +// stable. `ggml_backend_*` and `ggml_fp16_to_fp32` are stable too. +// +// Three pieces we rely on that aren't strictly guaranteed public: +// +// 1. Direct writes to `ggml_tensor::type`, `ne[]`, `nb[]` — the struct is +// public and fields are spec'd, but there's no sanctioned mutator for +// them post-creation. Used in set_tensor_type / set_tensor_shape / +// reclaim_slot_as. Risk: upstream could in principle introduce an +// opaque-tensor mode; in practice it hasn't in years. +// +// 2. `const_cast(gguf_get_tensor_name(...))` in rename_tensor. +// The pointer returned points into a mutable char[GGML_MAX_NAME] +// buffer inside a std::vector element. Defined behavior as long as +// upstream keeps name storage in-line (has done so forever). +// +// 3. `llama_model_loader` forward decl from src/llama-model-loader.h +// (internal, not llama.h). Only used as an opaque pointer key for +// the skip-prefix registry — we never dereference it. Could swap for +// `const void *` if upstream ever moved that type around. +// +// All three are trivially replaceable if upstream changes out from under +// us. See llama/compat/README.md for the escape hatches. + +#include +#include +#include +#include +#include +#include + +#include "ggml.h" +#include "ggml-backend.h" +#include "gguf.h" + +struct llama_model_loader; + +namespace llama_ollama_compat::detail { + +// -- gguf_context KV helpers -- +bool has_key(const gguf_context * meta, const char * key); +void copy_u32_kv(gguf_context * meta, const char * src, const char * dst); +void copy_f32_kv(gguf_context * meta, const char * src, const char * dst); +// Generic copy that preserves the source's gguf_type. Skips if `src` is +// missing or `dst` is already present. Arrays are copied verbatim +// (including element type). +void copy_kv(gguf_context * meta, const char * src, const char * dst); +// Copy every KV whose key starts with `old_prefix` to a new key under +// `new_prefix`. Old keys are left in place — harmless because the loader +// looks up keys by exact name and only queries the new prefix. +void rename_kv_prefix(gguf_context * meta, const char * old_prefix, + const char * new_prefix); +void inject_u32_if_missing (gguf_context * meta, const char * key, uint32_t v); +void inject_f32_if_missing (gguf_context * meta, const char * key, float v); +void inject_str_if_missing (gguf_context * meta, const char * key, const char * v); +void inject_bool_if_missing(gguf_context * meta, const char * key, bool v); +void inject_f32_arr_if_missing(gguf_context * meta, const char * key, + const float * data, size_t n); +void truncate_str_arr (gguf_context * meta, const char * key, size_t new_n); +void truncate_data_arr(gguf_context * meta, const char * key, + gguf_type elem_type, size_t elem_size, size_t new_n); + +// -- ggml_context tensor scans -- +bool any_tensor_with_prefix(const ggml_context * ctx, const char * prefix); + +// -- Tensor renaming / reshaping (mutates both gguf_context and ggml_context) -- +void rename_tensor(gguf_context * meta, ggml_context * ctx, + const char * old_name, const char * new_name); +void rename_tensors_containing(gguf_context * meta, ggml_context * ctx, + const char * needle, const char * replacement); +void set_tensor_type (ggml_tensor * t, ggml_type type); +void set_tensor_shape(ggml_tensor * t, std::initializer_list shape); +bool reclaim_slot_as (gguf_context * meta, ggml_context * ctx, + const char * orphan_name, const char * new_name, + std::initializer_list shape, ggml_type type); + +// -- File-offset capture (before rename) -- +size_t tensor_file_offset(const gguf_context * meta, const char * name); + +// -- Per-loader skip-prefix registry -- +void add_skip_prefix(const llama_model_loader * ml, std::string prefix); +bool should_skip_tensor_prefix(const llama_model_loader * ml, const char * name); + +// -- Per-loader "needs no-mmap" flag -- +// Handlers that register a load_op which transforms a TEXT-side tensor's +// bytes (e.g. concat reshape) must call disable_mmap_for(ml). With mmap +// the upstream loader binds the tensor directly to the file region, so +// our load_op has no writable buffer to fill. translate_metadata reads +// this flag and returns it back to the patch site. +void disable_mmap_for(const llama_model_loader * ml); +bool is_mmap_disabled_for(const llama_model_loader * ml); + +// -- Load-time transform registry -- +struct LoadOp { + std::function apply; + const char * description; +}; +void register_load_op(std::string dest_name, LoadOp op); +bool take_load_op (const char * dest_name, LoadOp & out); // removes + returns + +// Read `size` bytes at `offset` from `path` into `dst`. Used by LoadOps. +bool read_at(const char * path, size_t offset, void * dst, size_t size); + +// -- Common high-level transforms -- + +// F16 -> F32 promotion. Captures the source file offset at registration +// time so later renames/reshapes of this tensor don't invalidate the read. +void promote_tensor_to_f32(gguf_context * meta, ggml_context * ctx, const char * name); + +// Concatenate N source tensors into one destination. Captures each source's +// file offset + byte size at registration time. Layout assumption: sources +// concatenate cleanly along the destination's slow ggml axis, which in +// C order means the destination bytes are src[0] || src[1] || ... . +void register_concat_load(const gguf_context * meta, std::string dest_name, + const std::vector & src_names); + +// Mixed-type variant of register_concat_load: dequantizes each source to +// F32 via its ggml_type_traits.to_float and concatenates the F32 arrays. +// Use when sources differ in quantization (e.g. F16 q/k + Q8_0 v in some +// Ollama vision blobs). Caller must set the destination tensor's type to +// GGML_TYPE_F32 so dst_size matches the F32 concat size. +void register_concat_load_to_f32(const gguf_context * meta, + const ggml_context * ctx, + std::string dest_name, + const std::vector & src_names); + +} // namespace llama_ollama_compat::detail diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp new file mode 100644 index 000000000..14970110e --- /dev/null +++ b/llama/compat/llama-ollama-compat.cpp @@ -0,0 +1,2017 @@ +#include "llama-ollama-compat.h" +#include "llama-ollama-compat-util.h" + +#include "llama-impl.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace llama_ollama_compat { + +using namespace llama_ollama_compat::detail; // pull detail:: helpers into scope + +namespace { + +// Per-loader file path registry — set by translate_metadata, read by +// maybe_load_text_tensor so it can pass the path to load ops without a +// separate patch insertion in the model loader's load_all_data path. +std::mutex g_loader_path_mutex; +std::unordered_map g_loader_paths; + +// ========================================================================= +// gemma3 (text side) +// ========================================================================= + +// An Ollama-format gemma3 file declares arch="gemma3" AND exhibits at +// least one converter quirk. Different converter versions produced +// different quirks (4B/12B/27B have embedded vision + mm KVs; 1B uses +// non-standard rope key names; all of them omit layer_norm_rms_epsilon). +bool detect_ollama_gemma3(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "gemma3") != 0) return false; + + return has_key(meta, "gemma3.mm.tokens_per_image") + || any_tensor_with_prefix(ctx, "v.") + || any_tensor_with_prefix(ctx, "mm.") + || has_key(meta, "gemma3.rope.global.freq_base") + || has_key(meta, "gemma3.rope.local.freq_base") + || has_key(meta, "tokenizer.ggml.add_padding_token") + || has_key(meta, "tokenizer.ggml.add_unknown_token") + || !has_key(meta, "gemma3.attention.layer_norm_rms_epsilon"); +} + +void handle_gemma3(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_gemma3(meta, ctx)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format gemma3 GGUF; applying compatibility fixes\n", __func__); + + // Old Ollama converters sometimes used nested rope key names. Copy + // them to the flat names upstream expects BEFORE injecting defaults. + copy_f32_kv(meta, "gemma3.rope.global.freq_base", "gemma3.rope.freq_base"); + copy_f32_kv(meta, "gemma3.rope.local.freq_base", "gemma3.rope.freq_base_swa"); + + // Inject required KVs with their standard gemma3 defaults. + inject_f32_if_missing(meta, "gemma3.attention.layer_norm_rms_epsilon", 1e-6f); + inject_f32_if_missing(meta, "gemma3.rope.freq_base", 1000000.0f); + inject_f32_if_missing(meta, "gemma3.rope.freq_base_swa", 10000.0f); + + // Gemma3 4B/12B/27B ship with {type: "linear", factor: 8.0} rope scaling + // in their HF config to extend the 16k trained context to 131072. Ollama's + // old converter didn't write these. The 1B has no scaling — detect by + // context length. + const int64_t ctx_key = gguf_find_key(meta, "gemma3.context_length"); + if (ctx_key >= 0 && gguf_get_val_u32(meta, ctx_key) >= 131072) { + inject_str_if_missing(meta, "gemma3.rope.scaling.type", "linear"); + inject_f32_if_missing(meta, "gemma3.rope.scaling.factor", 8.0f); + } + + // Tokenizer vocab size vs embedding rows mismatch: Ollama leaves extra + // multimodal tokens (e.g. ) in the tokenizer arrays. + // Truncate to match token_embd rows so llama.cpp's dim check passes. + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) { + if (std::strcmp(ggml_get_name(t), "token_embd.weight") == 0) { + const size_t rows = t->ne[1]; // shape is [n_embd, n_vocab] + truncate_str_arr (meta, "tokenizer.ggml.tokens", rows); + truncate_data_arr(meta, "tokenizer.ggml.scores", GGUF_TYPE_FLOAT32, sizeof(float), rows); + truncate_data_arr(meta, "tokenizer.ggml.token_type", GGUF_TYPE_INT32, sizeof(int32_t), rows); + break; + } + } + + // Hide embedded vision tensors from the text loader. Ollama's Go side + // re-passes the same blob as --mmproj so the clip loader picks them up. + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); + + // Note: no RMSNorm weight shift needed. Ollama's published gemma3 blobs + // already have the +1 shift baked in, same as upstream's convert_hf. +} + +// ========================================================================= +// gemma3n (text side — vocab mismatch between token_embd and per_layer_token_embd) +// ========================================================================= +// +// Ollama publishes gemma-3n with the multimodal vocab (262400 tokens) for +// the main token_embd and tokenizer arrays, but the per-layer token embed +// only has the text vocab (262144 tokens). Upstream's loader expects both +// embedding tensors to have the same n_vocab, and reads n_vocab from the +// tokenizer.tokens array length — so the larger value wins and per_layer +// fails the dim check: +// +// tensor 'per_layer_token_embd.weight' has wrong shape; +// expected 8960, 262400, got 8960, 262144 +// +// Fix (mirroring handle_gemma3): truncate the tokenizer arrays AND the +// token_embd tensor's vocab dim down to the per_layer count. The dropped +// 256 entries are multimodal special tokens (image/audio markers); upstream +// gemma3n is text-only, so they're unused anyway. +// +// Note: tensor data isn't read at this point — the loader reads ggml_nbytes +// from the (newly-shrunk) tensor shape, so it just reads fewer rows from +// the same file offset. No load_op needed. + +bool detect_ollama_gemma3n(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "gemma3n") != 0) return false; + ggml_tensor * te = ggml_get_tensor(const_cast(ctx), "token_embd.weight"); + ggml_tensor * pe = ggml_get_tensor(const_cast(ctx), "per_layer_token_embd.weight"); + return te && pe && te->ne[1] != pe->ne[1]; +} + +void handle_gemma3n(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + (void) ml; + if (!detect_ollama_gemma3n(meta, ctx)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format gemma3n GGUF; truncating vocab to per_layer_token_embd size\n", __func__); + + ggml_tensor * pe = ggml_get_tensor(ctx, "per_layer_token_embd.weight"); + if (!pe) return; + const uint32_t target_vocab = (uint32_t) pe->ne[1]; + + if (ggml_tensor * t = ggml_get_tensor(ctx, "token_embd.weight")) { + set_tensor_shape(t, {t->ne[0], target_vocab}); + } + truncate_str_arr (meta, "tokenizer.ggml.tokens", target_vocab); + truncate_data_arr(meta, "tokenizer.ggml.scores", GGUF_TYPE_FLOAT32, sizeof(float), target_vocab); + truncate_data_arr(meta, "tokenizer.ggml.token_type", GGUF_TYPE_INT32, sizeof(int32_t), target_vocab); +} + +// ========================================================================= +// embeddinggemma (text side — sentence-transformer dense projection) +// ========================================================================= +// +// Ollama publishes embeddinggemma:300m with general.architecture=gemma3 and +// two extra dense layers stored as `dense.0.weight` / `dense.1.weight` +// (the sentence-transformers post-pooling projection that maps the 768-dim +// pooled embedding through 768→3072→768 for the matryoshka head). +// +// Upstream loads this model under arch=gemma-embedding, which: +// * disables causal attention (embeddings are bidirectional) +// * loads `dense_2.weight` and `dense_3.weight` by name (with shapes +// derived from gemma-embedding.dense_2_feat_in/out etc.) +// +// Without that arch, the gemma3 loader leaves dense.0/dense.1 unrequested +// and `done_getting_tensors` raises "wrong number of tensors" (2 unused). +// +// Detection: arch=gemma3 AND has dense.0.weight tensor (only embeddinggemma +// ships these — regular gemma3 chat models do not). +// Translation: switch arch_name to gemma-embedding, copy the gemma3.* KV +// prefix to gemma-embedding.*, derive dense_*_feat_* from the actual tensor +// shapes, and rename dense.0/dense.1 → dense_2/dense_3. + +bool detect_ollama_embeddinggemma(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "gemma3") != 0) return false; + return ggml_get_tensor(const_cast(ctx), "dense.0.weight") != nullptr; +} + +void handle_embeddinggemma(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + (void) ml; + if (!detect_ollama_embeddinggemma(meta, ctx)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format embeddinggemma; translating to gemma-embedding\n", __func__); + + // Switch architecture so upstream loads the embedding-specific code path + // (no causal attention, dense_2/dense_3 loaded by name). + arch_name = "gemma-embedding"; + gguf_set_val_str(meta, "general.architecture", "gemma-embedding"); + + // Mirror gemma3.* hparams under the new arch prefix. rename_kv_prefix + // copies (does not remove); the leftover gemma3.* keys are unused. + rename_kv_prefix(meta, "gemma3.", "gemma-embedding."); + + // Derive dense feat dims from the actual tensor shapes. + // dense.0.weight: [n_embd, dense_2_feat_out] + // dense.1.weight: [dense_3_feat_in, n_embd] + ggml_tensor * d0 = ggml_get_tensor(ctx, "dense.0.weight"); + ggml_tensor * d1 = ggml_get_tensor(ctx, "dense.1.weight"); + if (d0 && d1) { + gguf_set_val_u32(meta, "gemma-embedding.dense_2_feat_in", (uint32_t) d0->ne[0]); + gguf_set_val_u32(meta, "gemma-embedding.dense_2_feat_out", (uint32_t) d0->ne[1]); + gguf_set_val_u32(meta, "gemma-embedding.dense_3_feat_in", (uint32_t) d1->ne[0]); + gguf_set_val_u32(meta, "gemma-embedding.dense_3_feat_out", (uint32_t) d1->ne[1]); + } + + rename_tensor(meta, ctx, "dense.0.weight", "dense_2.weight"); + rename_tensor(meta, ctx, "dense.1.weight", "dense_3.weight"); +} + +// ========================================================================= +// qwen35moe (text side) +// ========================================================================= + +// Shared text-side fixes for Ollama-format qwen35 / qwen35moe GGUFs. +// Both arches use the same SSM-hybrid + M-RoPE + MTP+vision-monolithic +// converter quirks; only the arch name (and KV prefix) differs. +void apply_qwen35_text_fixes(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, const char * arch_prefix) { + auto kv = [arch_prefix](const char * suffix) { + return std::string(arch_prefix) + suffix; + }; + + // 1. attention.head_count_kv — upstream expects UINT32; Ollama wrote + // an array (one entry per layer, 0 for SSM layers, 2/4 for attention). + // Collapse to the max non-zero value. + { + const std::string key = kv(".attention.head_count_kv"); + const int64_t kid = gguf_find_key(meta, key.c_str()); + if (kid >= 0 && gguf_get_kv_type(meta, kid) == GGUF_TYPE_ARRAY) { + const size_t n = gguf_get_arr_n(meta, kid); + const auto * arr = static_cast(gguf_get_arr_data(meta, kid)); + uint32_t max_kv = 0; + for (size_t i = 0; i < n; ++i) if (arr[i] > max_kv) max_kv = arr[i]; + if (max_kv == 0) max_kv = 2; // safety fallback + gguf_remove_key (meta, key.c_str()); + gguf_set_val_u32 (meta, key.c_str(), max_kv); + } + } + + // 2. rope.dimension_sections — upstream expects a 4-element array + // (M-RoPE convention); Ollama wrote 3 elements. Pad with a trailing 0. + { + const std::string key = kv(".rope.dimension_sections"); + const int64_t kid = gguf_find_key(meta, key.c_str()); + if (kid >= 0 && gguf_get_arr_n(meta, kid) == 3) { + const auto * src = static_cast(gguf_get_arr_data(meta, kid)); + const int32_t padded[4] = { src[0], src[1], src[2], 0 }; + gguf_set_arr_data(meta, key.c_str(), GGUF_TYPE_INT32, padded, 4); + } + } + + // 3. Tensor rename: Ollama's `blk.N.ssm_dt` is upstream's + // `blk.N.ssm_dt.bias` (same shape). + { + std::vector targets; + const int64_t n = gguf_get_n_tensors(meta); + static const char suffix[] = ".ssm_dt"; + const size_t slen = sizeof(suffix) - 1; + for (int64_t i = 0; i < n; ++i) { + std::string name(gguf_get_tensor_name(meta, i)); + if (name.size() >= slen + && name.compare(name.size() - slen, slen, suffix) == 0) { + targets.push_back(std::move(name)); + } + } + for (const auto & from : targets) { + rename_tensor(meta, ctx, from.c_str(), (from + ".bias").c_str()); + } + } + + // 4. Drop embedded vision + MTP + projector tensors from the text loader. + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); + add_skip_prefix(ml, "mtp."); +} + +bool detect_ollama_qwen35moe(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "qwen35moe") != 0) return false; + + // Any Ollama-ism. Upstream qwen35moe files have none of these — the + // vision KVs live in a separate mmproj, MTP tensors are dropped, + // head_count_kv is a scalar, and the extra rope / ssm / feed_forward + // KVs are either absent or stored differently. + return has_key(meta, "qwen35moe.vision.block_count") + || has_key(meta, "qwen35moe.image_token_id") + || has_key(meta, "qwen35moe.ssm.v_head_reordered") + || has_key(meta, "qwen35moe.feed_forward_length") + || has_key(meta, "qwen35moe.rope.mrope_interleaved") + || any_tensor_with_prefix(ctx, "mtp.") + || any_tensor_with_prefix(ctx, "v."); +} + +void handle_qwen35moe(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_qwen35moe(meta, ctx)) return; + LLAMA_LOG_INFO("%s: detected Ollama-format qwen35moe GGUF; applying compatibility fixes\n", __func__); + apply_qwen35_text_fixes(ml, meta, ctx, "qwen35moe"); +} + +// ========================================================================= +// qwen35 (text side — non-MoE, e.g. qwen3.5:9b) +// ========================================================================= +// +// Same converter quirks as qwen35moe but the arch name has no "moe" suffix. +// All the SSM-hybrid / M-RoPE / MTP / monolithic-vision fix-ups apply. + +bool detect_ollama_qwen35(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "qwen35") != 0) return false; + return has_key(meta, "qwen35.vision.block_count") + || has_key(meta, "qwen35.image_token_id") + || has_key(meta, "qwen35.ssm.v_head_reordered") + || has_key(meta, "qwen35.rope.mrope_interleaved") + || any_tensor_with_prefix(ctx, "mtp.") + || any_tensor_with_prefix(ctx, "v."); +} + +void handle_qwen35(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_qwen35(meta, ctx)) return; + LLAMA_LOG_INFO("%s: detected Ollama-format qwen35 GGUF; applying compatibility fixes\n", __func__); + apply_qwen35_text_fixes(ml, meta, ctx, "qwen35"); +} + +// ========================================================================= +// gemma4 (text side) +// ========================================================================= +// +// Same arch name on both sides. Ollama publishes a monolithic GGUF that +// embeds the vision encoder + audio encoder + projector inline. Text-side +// KVs/tensor names match upstream verbatim — only fix is to hide the +// `a.*` / `v.*` / `mm.*` tensors from the text loader so n_tensors lines up. + +bool detect_ollama_gemma4(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "gemma4") != 0) return false; + return any_tensor_with_prefix(ctx, "a.") + || any_tensor_with_prefix(ctx, "v.") + || any_tensor_with_prefix(ctx, "mm."); +} + +void handle_gemma4(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_gemma4(meta, ctx)) return; + (void) ctx; + + LLAMA_LOG_INFO("%s: detected Ollama-format gemma4 GGUF; applying compatibility fixes\n", __func__); + + // Tokenizer fix: Ollama writes `tokenizer.ggml.model = 'llama'` (SPM) on + // gemma4 GGUFs, but gemma4 actually uses BPE — upstream-converted GGUFs + // use `'gemma4'` which selects LLAMA_VOCAB_TYPE_BPE in src/llama-vocab.cpp. + // With the wrong tokenizer type, gemma4's special tokens (e.g. + // `<|thought|>`, `<|turn>`, `<|channel>`) get split into multiple SPM + // subword pieces, so when the model emits them they come out as raw + // text instead of being recognized as control tokens. + // + // Ollama already supplies `tokenizer.ggml.merges` (needed for BPE) and + // `tokenizer.ggml.pre = 'gemma4'`, so flipping the model name is enough. + { + const int64_t kid = gguf_find_key(meta, "tokenizer.ggml.model"); + if (kid >= 0) { + const char * cur = gguf_get_val_str(meta, kid); + if (cur && std::strcmp(cur, "llama") == 0) { + gguf_set_val_str(meta, "tokenizer.ggml.model", "gemma4"); + } + } + } + + // Hide embedded audio + vision + projector tensors from the text loader. + add_skip_prefix(ml, "a."); + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// deepseek-ocr (text side) +// ========================================================================= +// +// Ollama uses arch name "deepseekocr" / KV prefix "deepseekocr.*". +// Upstream uses "deepseek2-ocr" (with hyphen) / "deepseek2-ocr.*". +// +// Aside from the prefix rename: +// * Inject `expert_feed_forward_length` from the per-expert ffn_down_exps +// shape (Ollama omits it; the value is the inner FFN dim of one expert, +// 896 for the 3B model). +// * Inject `expert_shared_count` from the ffn_down_shexp shape (Ollama +// omits it; the shared experts share their FFN dim with regular experts, +// so count = shexp_dim / expert_feed_forward_length). +// * Skip embedded vision (`v.*`), projector (`mm.*`), and the SAM encoder +// (`s.*`) tensors from the text loader. + +bool detect_ollama_deepseekocr(const gguf_context * meta) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + return std::strcmp(gguf_get_val_str(meta, arch_kid), "deepseekocr") == 0; +} + +void handle_deepseekocr(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + if (!detect_ollama_deepseekocr(meta)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format deepseekocr GGUF; applying compatibility fixes\n", __func__); + + gguf_set_val_str(meta, "general.architecture", "deepseek2-ocr"); + rename_kv_prefix(meta, "deepseekocr.", "deepseek2-ocr."); + arch_name = "deepseek2-ocr"; + + // Inject defaults Ollama omitted entirely. + inject_f32_if_missing(meta, "deepseek2-ocr.attention.layer_norm_rms_epsilon", + 1e-6f); + + // Recover expert_feed_forward_length from blk.1 (first MoE block; blk.0 + // is dense). ne[0] of ffn_down_exps is the per-expert inner dim. + if (!has_key(meta, "deepseek2-ocr.expert_feed_forward_length")) { + if (ggml_tensor * t = ggml_get_tensor(ctx, "blk.1.ffn_down_exps.weight")) { + gguf_set_val_u32(meta, "deepseek2-ocr.expert_feed_forward_length", + (uint32_t) t->ne[0]); + } + } + + // Recover expert_shared_count from blk.1.ffn_down_shexp shape. + // shape ne[0] = expert_shared_count * expert_feed_forward_length + if (!has_key(meta, "deepseek2-ocr.expert_shared_count")) { + ggml_tensor * shexp = ggml_get_tensor(ctx, "blk.1.ffn_down_shexp.weight"); + const int64_t fflen_kid = gguf_find_key(meta, "deepseek2-ocr.expert_feed_forward_length"); + if (shexp && fflen_kid >= 0) { + const uint32_t fflen = gguf_get_val_u32(meta, fflen_kid); + if (fflen > 0) { + gguf_set_val_u32(meta, "deepseek2-ocr.expert_shared_count", + (uint32_t)(shexp->ne[0] / fflen)); + } + } + } + + // Hide embedded SAM (`s.*`), vision (`v.*`), and projector (`mm.*`) + // tensors from the text loader. + add_skip_prefix(ml, "s."); + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// nemotron_h_moe (text only) +// ========================================================================= +// +// Same arch name on both sides. Most variants (e.g. nemotron-cascade-2) +// load as-is. The latent-FFN variants (e.g. nemotron-3-super 120B-A12B) +// rename `ffn_latent_in` / `ffn_latent_out` to `ffn_latent_down` / +// `ffn_latent_up`, and need `moe_latent_size` injected (derived from +// the latent tensor shape). + +bool detect_ollama_nemotron_h_moe(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "nemotron_h_moe") != 0) return false; + return any_tensor_with_prefix(ctx, "blk.1.ffn_latent_in") + || any_tensor_with_prefix(ctx, "blk.0.ffn_latent_in") + || any_tensor_with_prefix(ctx, "mtp."); +} + +void handle_nemotron_h_moe(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_nemotron_h_moe(meta, ctx)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format nemotron_h_moe GGUF; applying compatibility fixes\n", __func__); + + // Inject moe_latent_size for latent-FFN variants (e.g. super 120B-A12B). + // Standard variants (e.g. cascade-2 30B-A3B) have no latent tensors and + // use n_embd as the MoE inner dim — leave the key absent. + if (!has_key(meta, "nemotron_h_moe.moe_latent_size")) { + for (uint32_t b = 0; b < 1024; ++b) { + char name[64]; + std::snprintf(name, sizeof(name), "blk.%u.ffn_latent_in.weight", b); + if (ggml_tensor * t = ggml_get_tensor(ctx, name)) { + gguf_set_val_u32(meta, "nemotron_h_moe.moe_latent_size", + (uint32_t) t->ne[1]); + break; + } + } + } + + // Rename the latent projection tensors to upstream's naming (no-op when + // the file has no latent tensors). + rename_tensors_containing(meta, ctx, ".ffn_latent_in", ".ffn_latent_down"); + rename_tensors_containing(meta, ctx, ".ffn_latent_out", ".ffn_latent_up"); + + // Drop MTP (Multi-Token Prediction) tensors — Ollama's converter emits + // them as one-tensor-per-expert (`mtp.layers.X.mixer.experts.Y.{up,down}_proj`) + // which upstream's nemotron_h_moe loader doesn't claim. Total: ~1040 extra + // tensors on super 120B. + add_skip_prefix(ml, "mtp."); +} + +// ========================================================================= +// llama4 (text side) +// ========================================================================= +// +// Same arch name on both sides. Ollama publishes a monolithic GGUF that +// embeds the vision encoder + projector inline. Text-side KVs/tensor +// names match upstream verbatim — only fix is to hide `v.*`/`mm.*` from +// the text loader so n_tensors lines up. + +bool detect_ollama_llama4(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "llama4") != 0) return false; + return any_tensor_with_prefix(ctx, "v.") + || any_tensor_with_prefix(ctx, "mm."); +} + +void handle_llama4(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_llama4(meta, ctx)) return; + (void) meta; + (void) ctx; + + LLAMA_LOG_INFO("%s: detected Ollama-format llama4 GGUF; applying compatibility fixes\n", __func__); + + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// glm-ocr (text side) +// ========================================================================= +// +// Ollama uses arch name "glmocr" / KV prefix "glmocr.*" with 16 blocks. +// Upstream uses "glm4" / "glm4.*" — the GLM-OCR variant of LLM_ARCH_GLM4 +// is identified by `n_layer = 17` (16 main + 1 nextn predict layer). +// Ollama drops the nextn layer entirely, so we report n_layer = 16 and +// leave `nextn_predict_layers` absent (defaults to 0 = no nextn path). +// +// Bigger surgery: GLM4 expects fused gate+up MLP weights stored at +// `blk.X.ffn_up.weight` with shape `[n_embd, n_ff*2]`. Ollama writes +// the gate and up halves as separate `ffn_gate.weight` / `ffn_up.weight` +// tensors (each `[n_embd, n_ff]`). We register a concat load op that +// reads gate+up bytes and stitches them into the fused upstream slot. + +// Per-block: register a concat load that fuses Ollama's separate +// ffn_gate + ffn_up into upstream's single `blk.X.ffn_up.weight` +// tensor with doubled out dim. Capture source file offsets BEFORE any +// renames invalidate them (same pattern as qwen35moe QKV merge). +void register_glm4_ffn_concat(gguf_context * meta, ggml_context * ctx, int block_idx) { + char gate_n[64], up_n[64]; + std::snprintf(gate_n, sizeof(gate_n), "blk.%d.ffn_gate.weight", block_idx); + std::snprintf(up_n, sizeof(up_n), "blk.%d.ffn_up.weight", block_idx); + + if (!ggml_get_tensor(ctx, gate_n) || !ggml_get_tensor(ctx, up_n)) return; + + // GLM4's fused ffn_up has gate as first half, up as second half + // (so ggml_swiglu's silu(first_half) * second_half gives silu(gate) * up). + register_concat_load(meta, up_n, {gate_n, up_n}); + + if (ggml_tensor * t = ggml_get_tensor(ctx, up_n)) { + set_tensor_shape(t, {t->ne[0], t->ne[1] * 2}); + } +} + +bool detect_ollama_glmocr(const gguf_context * meta) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + return std::strcmp(gguf_get_val_str(meta, arch_kid), "glmocr") == 0; +} + +void handle_glmocr(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + if (!detect_ollama_glmocr(meta)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format glmocr GGUF; applying compatibility fixes\n", __func__); + + gguf_set_val_str(meta, "general.architecture", "glm4"); + rename_kv_prefix(meta, "glmocr.", "glm4."); + arch_name = "glm4"; + + // M-RoPE: Ollama writes a 3-element `rope.mrope_section`, upstream expects + // a 4-element `rope.dimension_sections` (pad trailing 0). + { + const int64_t kid = gguf_find_key(meta, "glm4.rope.mrope_section"); + if (kid >= 0 && gguf_get_arr_n(meta, kid) == 3) { + const auto * src = static_cast(gguf_get_arr_data(meta, kid)); + const int32_t padded[4] = { src[0], src[1], src[2], 0 }; + gguf_set_arr_data(meta, "glm4.rope.dimension_sections", + GGUF_TYPE_INT32, padded, 4); + } + } + // Inject `rope.dimension_count` from key_length (used as the rope dim). + if (!has_key(meta, "glm4.rope.dimension_count")) { + const int64_t kid = gguf_find_key(meta, "glm4.attention.key_length"); + if (kid >= 0) { + gguf_set_val_u32(meta, "glm4.rope.dimension_count", + gguf_get_val_u32(meta, kid)); + } + } + + // Tokenizer pre-tokenizer: Ollama wrote `llama-bpe`, but glm-ocr uses + // `chatglm-bpe` (different regex split — wrong pre-tokenization can + // fragment GLM's special tokens). + { + const int64_t kid = gguf_find_key(meta, "tokenizer.ggml.pre"); + if (kid >= 0) { + const char * cur = gguf_get_val_str(meta, kid); + if (cur && std::strcmp(cur, "chatglm-bpe") != 0) { + gguf_set_val_str(meta, "tokenizer.ggml.pre", "chatglm-bpe"); + } + } + } + + // Tensor renames (substring): each leaf appears once per block and + // doesn't overlap the others. + rename_tensors_containing(meta, ctx, ".attn_out", ".attn_output"); + rename_tensors_containing(meta, ctx, ".post_attn_norm", ".post_attention_norm"); + rename_tensors_containing(meta, ctx, ".post_ffn_norm", ".post_ffw_norm"); + + // Fuse ffn_gate + ffn_up → ffn_up[:, 2*n_ff] for every block, then mark + // the orphan ffn_gate tensors as skip so n_tensors lines up. + // + // The concat reshape grows ne[1] of ffn_up from N to 2N, so the file's + // mmap region for the original tensor is too small to back it. Force + // the loader off the mmap path so it pre-allocates real backend buffers + // that our register_concat_load can fill at load_all_data time. + disable_mmap_for(ml); + { + const int64_t n_blk_kid = gguf_find_key(meta, "glm4.block_count"); + const uint32_t n_blocks = n_blk_kid >= 0 ? gguf_get_val_u32(meta, n_blk_kid) : 16; + for (uint32_t b = 0; b < n_blocks; ++b) { + register_glm4_ffn_concat(meta, ctx, (int) b); + char skip_pref[64]; + std::snprintf(skip_pref, sizeof(skip_pref), "blk.%u.ffn_gate.", b); + add_skip_prefix(ml, skip_pref); + } + } + + // Hide embedded vision + projector tensors from the text loader. + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// gpt-oss (text only) +// ========================================================================= +// +// Ollama uses arch name "gptoss" (no hyphen) and KV prefix "gptoss.*". +// Upstream uses "gpt-oss" / "gpt-oss.*". Same tensor layout otherwise, +// except: +// * `blk.X.attn_sinks` -> `blk.X.attn_sinks.weight` (missing suffix) +// * `blk.X.ffn_norm.weight` -> `blk.X.post_attention_norm.weight` +// (the second-norm-per-block names differ between converters) + +bool detect_ollama_gptoss(const gguf_context * meta) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + return std::strcmp(gguf_get_val_str(meta, arch_kid), "gptoss") == 0; +} + +// `arch_name` is mutated to "gpt-oss" so the caller's subsequent +// LLM_KV lookups query the renamed prefix. +void handle_gptoss(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + if (!detect_ollama_gptoss(meta)) return; + (void) ml; + + LLAMA_LOG_INFO("%s: detected Ollama-format gpt-oss GGUF; applying compatibility fixes\n", __func__); + + gguf_set_val_str(meta, "general.architecture", "gpt-oss"); + rename_kv_prefix(meta, "gptoss.", "gpt-oss."); + arch_name = "gpt-oss"; + + // Upstream's gpt-oss loader requires `gpt-oss.expert_feed_forward_length` + // (n_ff_exp). Ollama omitted it; recover from the ffn_gate_exps tensor + // shape — for gpt-oss the tensor is created as {n_embd, n_ff_exp, n_expert} + // so ne[1] is the per-expert FFN dim. + if (!has_key(meta, "gpt-oss.expert_feed_forward_length")) { + if (ggml_tensor * t = ggml_get_tensor(ctx, "blk.0.ffn_gate_exps.weight")) { + gguf_set_val_u32(meta, "gpt-oss.expert_feed_forward_length", (uint32_t) t->ne[1]); + } + } + + // Tensor renames. `rename_tensors_containing` does a substring replace + // on first occurrence — each needle below appears exactly once per + // tensor name and the needles don't overlap each other. + rename_tensors_containing(meta, ctx, ".attn_out", + ".attn_output"); // wo: out -> output + rename_tensors_containing(meta, ctx, ".attn_sinks", + ".attn_sinks.weight"); // add missing suffix + rename_tensors_containing(meta, ctx, ".ffn_norm", + ".post_attention_norm"); +} + +// ========================================================================= +// lfm2 (text only) +// ========================================================================= +// +// Same arch name ("lfm2") on both sides. Only difference is the +// pre-output-projection norm: Ollama writes `output_norm.weight`, +// upstream writes `token_embd_norm.weight` (with the LFM2-specific +// LLM_TENSOR_OUTPUT_NORM_LFM2 mapping). One tensor rename. + +bool detect_ollama_lfm2(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "lfm2") != 0) return false; + // Marker: Ollama-converted lfm2 has output_norm.weight, upstream has + // token_embd_norm.weight instead. + return ggml_get_tensor(const_cast(ctx), "output_norm.weight") != nullptr + && ggml_get_tensor(const_cast(ctx), "token_embd_norm.weight") == nullptr; +} + +void handle_lfm2(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_lfm2(meta, ctx)) return; + (void) ml; + + LLAMA_LOG_INFO("%s: detected Ollama-format lfm2 GGUF; applying compatibility fixes\n", __func__); + + rename_tensor(meta, ctx, "output_norm.weight", "token_embd_norm.weight"); + + // Older Ollama converters wrote a stale `lfm2.feed_forward_length` that + // didn't match the actual ffn_gate tensor shape (e.g. claimed 12288 on + // a model whose ffn_gate is [2048, 8192]). Fix from the tensor shape. + if (ggml_tensor * t = ggml_get_tensor(ctx, "blk.0.ffn_gate.weight")) { + const uint32_t real_n_ff = (uint32_t) t->ne[1]; + const int64_t kid = gguf_find_key(meta, "lfm2.feed_forward_length"); + if (kid < 0 || gguf_get_val_u32(meta, kid) != real_n_ff) { + gguf_set_val_u32(meta, "lfm2.feed_forward_length", real_n_ff); + } + } +} + +// ========================================================================= +// mistral3 (text only — for now) +// ========================================================================= +// +// Same arch name on both sides. Ollama publishes a monolithic GGUF that +// embeds the vision encoder + projector inline, similar to gemma3 and +// qwen35moe. Differences this handler addresses: +// +// * Embedded `v.*` / `mm.*` tensors must be hidden from the text +// loader (otherwise n_tensors mismatch). +// * RoPE YaRN parameters use unprefixed names: Ollama writes +// `rope.scaling.beta_fast`/`beta_slow`, upstream wants +// `rope.scaling.yarn_beta_fast`/`yarn_beta_slow`. +// * Attention temperature scale: Ollama writes `rope.scaling_beta`, +// upstream reads `attention.temperature_scale`. Same numeric value. +// +// Vision/clip translation is not implemented yet — the user has to skip +// `--mmproj` until a clip handler lands. + +bool detect_ollama_mistral3(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "mistral3") != 0) return false; + // Marker: Ollama-style monolithic file embeds v.*/mm.* tensors; + // upstream HF mistral3 ships these in a separate mmproj. + return any_tensor_with_prefix(ctx, "v.") + || any_tensor_with_prefix(ctx, "mm.") + || has_key(meta, "mistral3.rope.scaling.beta_fast") + || has_key(meta, "mistral3.rope.scaling_beta"); +} + +void handle_mistral3(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_mistral3(meta, ctx)) return; + (void) ctx; + + LLAMA_LOG_INFO("%s: detected Ollama-format mistral3 GGUF; applying compatibility fixes\n", __func__); + + // RoPE YaRN parameter renames. + copy_kv(meta, "mistral3.rope.scaling.beta_fast", + "mistral3.rope.scaling.yarn_beta_fast"); + copy_kv(meta, "mistral3.rope.scaling.beta_slow", + "mistral3.rope.scaling.yarn_beta_slow"); + // Attention temperature scale: same value, different home. + copy_kv(meta, "mistral3.rope.scaling_beta", + "mistral3.attention.temperature_scale"); + + // Hide embedded vision + projector tensors from the text loader. + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// glm4moelite (text side — GLM-4.x-Flash, arch translation to deepseek2) +// ========================================================================= +// +// Ollama publishes GLM-4.7-Flash (and similar Flash variants) with +// general.architecture=glm4moelite using DeepSeek-V2 style MLA attention, +// but with the older convention of writing PER-HEAD key/value dims. +// Upstream collapsed all of these into the deepseek2 arch with the +// MLA-absorbed convention (head_count_kv=1, key/value dims = the +// kv_lora_rank-relative absorbed sizes). +// +// Tensor structure is identical to deepseek2 (844 tensors, exact name +// match including attn_kv_a_mqa, attn_k_b, attn_v_b, attn_q_a/b, etc.) — +// only KV semantics differ. +// +// Translation: +// * arch_name: glm4moelite → deepseek2 +// * KV prefix: glm4moelite.* → deepseek2.* +// * head_count_kv: original num_kv_heads (e.g. 20) → 1 (MLA absorbed) +// * key_length: head_dim → kv_lora_rank + rope.dimension_count +// (e.g. 256 → 512+64=576) +// * value_length: head_dim → kv_lora_rank +// (e.g. 256 → 512) +// * key_length_mla / value_length_mla: were the absorbed dims (576/512); +// upstream's _mla variants are per-head dims +// (256/256). Swap to head_dim. +// * expert_group_count / expert_group_used_count: required by deepseek2 +// loader; default to 1 (no group routing). + +bool detect_ollama_glm4moelite(const gguf_context * meta) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + return std::strcmp(gguf_get_val_str(meta, arch_kid), "glm4moelite") == 0; +} + +void handle_glm4moelite(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + (void) ml; + (void) ctx; + if (!detect_ollama_glm4moelite(meta)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format glm4moelite GGUF; translating to deepseek2 (MLA conventions)\n", __func__); + + arch_name = "deepseek2"; + gguf_set_val_str(meta, "general.architecture", "deepseek2"); + + // Mirror glm4moelite.* hparams under deepseek2.* (rename copies; the + // original glm4moelite.* keys remain but are unread — only used as the + // "original head_dim" source below). + rename_kv_prefix(meta, "glm4moelite.", "deepseek2."); + + // MLA absorbs all KV heads — upstream uses 1. + gguf_set_val_u32(meta, "deepseek2.attention.head_count_kv", 1); + + // key/value lengths to MLA-absorbed dims, derived from kv_lora_rank + // and rope.dimension_count (both already mirrored under deepseek2.*). + { + const int64_t kv_lora_kid = gguf_find_key(meta, "deepseek2.attention.kv_lora_rank"); + const int64_t rope_kid = gguf_find_key(meta, "deepseek2.rope.dimension_count"); + if (kv_lora_kid >= 0 && rope_kid >= 0) { + const uint32_t kv_lora = gguf_get_val_u32(meta, kv_lora_kid); + const uint32_t rope_d = gguf_get_val_u32(meta, rope_kid); + gguf_set_val_u32(meta, "deepseek2.attention.key_length", kv_lora + rope_d); + gguf_set_val_u32(meta, "deepseek2.attention.value_length", kv_lora); + } + } + + // *_mla variants: upstream wants per-head dims (head_dim). Read original + // head_dim from the un-renamed glm4moelite.attention.key_length (which + // held the per-head dim in Ollama's convention). + { + const int64_t hd_kid = gguf_find_key(meta, "glm4moelite.attention.key_length"); + if (hd_kid >= 0) { + const uint32_t head_dim = gguf_get_val_u32(meta, hd_kid); + gguf_set_val_u32(meta, "deepseek2.attention.key_length_mla", head_dim); + gguf_set_val_u32(meta, "deepseek2.attention.value_length_mla", head_dim); + } + } + + // DeepSeek-V3 expert grouping; GLM-4-MoE-Lite doesn't use it but the + // loader expects the keys to be present. Default to 1. + inject_u32_if_missing(meta, "deepseek2.expert_group_count", 1); + inject_u32_if_missing(meta, "deepseek2.expert_group_used_count", 1); +} + +// ========================================================================= +// qwen25vl (text side — Qwen2.5-VL, arch translation to qwen2vl) +// ========================================================================= +// +// Ollama publishes Qwen2.5-VL with general.architecture=qwen25vl, but +// upstream loads both Qwen2-VL and Qwen2.5-VL under arch=qwen2vl +// (which reads the rope.dimension_sections KV for M-RoPE). +// +// Translation: +// * arch_name: qwen25vl → qwen2vl (loader uses arch as KV prefix) +// * KV prefix: qwen25vl.* → qwen2vl.* +// * rope.mrope_section (3 elements) → rope.dimension_sections (4, padded with 0) +// * Hide vision+projector tensors from the text loader. + +bool detect_ollama_qwen25vl(const gguf_context * meta) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + return std::strcmp(gguf_get_val_str(meta, arch_kid), "qwen25vl") == 0; +} + +void handle_qwen25vl(const llama_model_loader * ml, gguf_context * meta, + ggml_context * ctx, std::string & arch_name) { + (void) ctx; + if (!detect_ollama_qwen25vl(meta)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format qwen25vl GGUF; translating to qwen2vl\n", __func__); + + // Switch architecture so the loader reads qwen2vl.* keys (and uses the + // qwen2vl model build path, which handles M-RoPE). + arch_name = "qwen2vl"; + gguf_set_val_str(meta, "general.architecture", "qwen2vl"); + + // Mirror the qwen25vl.* KVs under qwen2vl.* (rename_kv_prefix copies; + // the original qwen25vl.* keys remain but are unread). + rename_kv_prefix(meta, "qwen25vl.", "qwen2vl."); + + // Translate mrope_section (3 elems) → dimension_sections (4 elems, padded). + const int64_t kid = gguf_find_key(meta, "qwen2vl.rope.mrope_section"); + if (kid >= 0 && gguf_get_arr_n(meta, kid) >= 3) { + const auto * src = static_cast(gguf_get_arr_data(meta, kid)); + const int32_t padded[4] = { src[0], src[1], src[2], 0 }; + gguf_set_arr_data(meta, "qwen2vl.rope.dimension_sections", + GGUF_TYPE_INT32, padded, 4); + } + + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// qwen3vl (text side — Qwen3-VL) +// ========================================================================= +// +// Ollama publishes Qwen3-VL with general.architecture=qwen3vl (matches +// upstream). Two missing KVs that the upstream qwen3vl loader requires: +// +// * qwen3vl.rope.dimension_sections — M-RoPE section sizes. Derived from +// the HF config (rope_scaling.mrope_section). Hardcoded here as +// [24, 20, 20, 0] which matches Qwen3-VL-8B (head_dim=128, sum=64). +// If new Qwen3-VL variants ship with a different mrope, derive from +// the head_dim or read from a published KV. +// +// * qwen3vl.n_deepstack_layers — count of deepstack adapters. Length of +// qwen3vl.vision.deepstack_visual_indexes (3 for Qwen3-VL-8B). + +bool detect_ollama_qwen3vl(const gguf_context * meta, const ggml_context * ctx) { + (void) ctx; + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0) return false; + if (std::strcmp(gguf_get_val_str(meta, arch_kid), "qwen3vl") != 0) return false; + // Marker: upstream-converted qwen3vl always has rope.dimension_sections; + // Ollama's blob doesn't. + return !has_key(meta, "qwen3vl.rope.dimension_sections"); +} + +void handle_qwen3vl(const llama_model_loader * ml, gguf_context * meta, ggml_context * ctx) { + (void) ctx; + if (!detect_ollama_qwen3vl(meta, ctx)) return; + + LLAMA_LOG_INFO("%s: detected Ollama-format qwen3vl GGUF; applying compatibility fixes\n", __func__); + + // Inject required M-RoPE sections (Qwen3-VL-8B default). + const int32_t mrope[4] = { 24, 20, 20, 0 }; + gguf_set_arr_data(meta, "qwen3vl.rope.dimension_sections", + GGUF_TYPE_INT32, mrope, 4); + + // Derive n_deepstack_layers from the deepstack indexes array length. + const int64_t ds_kid = gguf_find_key(meta, "qwen3vl.vision.deepstack_visual_indexes"); + const uint32_t n_ds = (ds_kid >= 0) ? (uint32_t) gguf_get_arr_n(meta, ds_kid) : 0; + inject_u32_if_missing(meta, "qwen3vl.n_deepstack_layers", n_ds); + + add_skip_prefix(ml, "v."); + add_skip_prefix(ml, "mm."); +} + +// ========================================================================= +// gemma3 (clip side) +// ========================================================================= + +constexpr std::pair kGemma3ClipRenames[] = { + {"v.patch_embedding", "v.patch_embd"}, + {"v.position_embedding", "v.position_embd"}, + {"v.post_layernorm", "v.post_ln"}, + {".layer_norm1", ".ln1"}, + {".layer_norm2", ".ln2"}, + {".attn_output", ".attn_out"}, + {".mlp.fc1", ".ffn_down"}, + {".mlp.fc2", ".ffn_up"}, + {"mm.mm_input_projection", "mm.input_projection"}, + {"mm.mm_soft_emb_norm", "mm.soft_emb_norm"}, +}; + +void handle_gemma3_clip(gguf_context * meta, ggml_context * ctx) { + copy_u32_kv(meta, "gemma3.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "gemma3.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "gemma3.vision.feed_forward_length", "clip.vision.feed_forward_length"); + copy_u32_kv(meta, "gemma3.vision.image_size", "clip.vision.image_size"); + copy_u32_kv(meta, "gemma3.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "gemma3.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_f32_kv(meta, "gemma3.vision.attention.layer_norm_epsilon", "clip.vision.attention.layer_norm_epsilon"); + // projection_dim = text model's embedding_length (mmproj out == LM in). + copy_u32_kv(meta, "gemma3.embedding_length", "clip.vision.projection_dim"); + + static const float kHalfHalfHalf[3] = {0.5f, 0.5f, 0.5f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kHalfHalfHalf, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kHalfHalfHalf, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_gelu", true); + gguf_set_val_str(meta, "clip.projector_type", "gemma3"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + for (const auto & [from, to] : kGemma3ClipRenames) { + rename_tensors_containing(meta, ctx, from, to); + } + + // Upstream stores patch_embd/position_embd as F32 (Gemma3VisionModel + // tensor_force_quant); Ollama stored F16. Metal's IM2COL convolution + // requires F32, so promote both at load time. + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); + promote_tensor_to_f32(meta, ctx, "v.position_embd.weight"); +} + +// ========================================================================= +// qwen35moe (clip side) +// ========================================================================= + +constexpr std::pair kQwen35moeClipRenames[] = { + {"v.pos_embed", "v.position_embd"}, + {"v.patch_embed", "v.patch_embd"}, + {"v.merger.norm", "v.post_ln"}, + {"v.merger.linear_fc1", "mm.0"}, + {"v.merger.linear_fc2", "mm.2"}, + {".mlp.linear_fc1", ".ffn_up"}, + {".mlp.linear_fc2", ".ffn_down"}, + {".norm1", ".ln1"}, + {".norm2", ".ln2"}, +}; + +// Register a QKV merge for a single vision block: Ollama has separate +// attn_q, attn_k, attn_v tensors; upstream wants them concatenated along +// their slow axis. Capture source file offsets BEFORE renaming attn_q. +void register_qwen35moe_qkv_merge(gguf_context * meta, ggml_context * ctx, int block_idx) { + char q[64], k[64], v[64], qbias[64], kbias[64], vbias[64], qkv_w[64], qkv_b[64]; + std::snprintf(q, sizeof(q), "v.blk.%d.attn_q.weight", block_idx); + std::snprintf(k, sizeof(k), "v.blk.%d.attn_k.weight", block_idx); + std::snprintf(v, sizeof(v), "v.blk.%d.attn_v.weight", block_idx); + std::snprintf(qbias, sizeof(qbias), "v.blk.%d.attn_q.bias", block_idx); + std::snprintf(kbias, sizeof(kbias), "v.blk.%d.attn_k.bias", block_idx); + std::snprintf(vbias, sizeof(vbias), "v.blk.%d.attn_v.bias", block_idx); + std::snprintf(qkv_w, sizeof(qkv_w), "v.blk.%d.attn_qkv.weight", block_idx); + std::snprintf(qkv_b, sizeof(qkv_b), "v.blk.%d.attn_qkv.bias", block_idx); + + if (!ggml_get_tensor(ctx, q)) return; // no vision block at this index + + // Capture source offsets for the concat BEFORE renaming. + register_concat_load(meta, qkv_w, {q, k, v}); + register_concat_load(meta, qkv_b, {qbias, kbias, vbias}); + + // Rename attn_q -> attn_qkv and widen from [hidden, hidden] to [hidden, 3*hidden]. + rename_tensor(meta, ctx, q, qkv_w); + if (ggml_tensor * t = ggml_get_tensor(ctx, qkv_w)) set_tensor_shape(t, {t->ne[0], t->ne[1] * 3}); + rename_tensor(meta, ctx, qbias, qkv_b); + if (ggml_tensor * t = ggml_get_tensor(ctx, qkv_b)) set_tensor_shape(t, {t->ne[0] * 3}); +} + +// Register the patch_embed reshape + split + F16->F32. +// +// Source: one Ollama tensor `v.patch_embed.weight`, ggml shape +// [h=16, w=16, t=2, packed=3456] F16 +// where `packed` is the PyTorch row-major flattening of HF's +// [out_c=1152, in_c=3, ...] dim pair, so packed_c = c_out*3 + c_in. +// +// Destination: two upstream tensors with ggml shape +// [h=16, w=16, c_in=3, c_out=1152] F32 each, one per temporal slice. +// +// For each output element (h, w, c_in, c_out): +// src_idx = h + w*W + t*W*H + (c_out*C_in + c_in)*W*H*T +// dst_idx = h + w*W + c_in*W*H + c_out*W*H*C_in +void register_qwen35moe_patch_embed_split(gguf_context * meta, ggml_context * ctx) { + const char * src_name = "v.patch_embed.weight"; + if (gguf_find_tensor(meta, src_name) < 0) return; + const ggml_tensor * src_t = ggml_get_tensor(ctx, src_name); + if (!src_t) return; + + const size_t src_offset = tensor_file_offset(meta, src_name); + const size_t src_size = ggml_nelements(src_t) * sizeof(uint16_t); + + constexpr int H = 16, W = 16, T = 2, CIN = 3, COUT = 1152; + constexpr size_t HW = (size_t) H * W; + + auto make_slice_op = [=](int slice_idx) { + return LoadOp{ + [=](const char * path, void * dst, size_t dst_size) { + if (dst_size != (size_t) H * W * CIN * COUT * sizeof(float)) return false; + std::vector src(src_size); + if (!read_at(path, src_offset, src.data(), src_size)) return false; + const uint16_t * sp = reinterpret_cast(src.data()); + float * dp = reinterpret_cast(dst); + for (int c_out = 0; c_out < COUT; ++c_out) { + for (int c_in = 0; c_in < CIN; ++c_in) { + const size_t packed = (size_t) c_out * CIN + c_in; + const uint16_t * in_base = sp + HW * (slice_idx + T * packed); + float * out_base = dp + HW * (c_in + CIN * c_out); + for (size_t i = 0; i < HW; ++i) out_base[i] = ggml_fp16_to_fp32(in_base[i]); + } + } + return true; + }, + slice_idx == 0 ? "patch_embed slice 0 (permute+F16->F32)" + : "patch_embed slice 1 (permute+F16->F32)", + }; + }; + + // Rename src -> `v.patch_embd.weight`, reshape to dest layout, register + // the slice-0 load op. + rename_tensor(meta, ctx, src_name, "v.patch_embd.weight"); + if (ggml_tensor * dest0 = ggml_get_tensor(ctx, "v.patch_embd.weight")) { + set_tensor_shape(dest0, {H, W, CIN, COUT}); + set_tensor_type (dest0, GGML_TYPE_F32); + } + register_load_op("v.patch_embd.weight", make_slice_op(0)); + + // Reclaim the `v.blk.0.attn_k.weight` slot (orphaned by the QKV merge) + // as the sibling `v.patch_embd.weight.1`. + reclaim_slot_as(meta, ctx, + "v.blk.0.attn_k.weight", "v.patch_embd.weight.1", + {H, W, CIN, COUT}, GGML_TYPE_F32); + register_load_op("v.patch_embd.weight.1", make_slice_op(1)); +} + +void handle_qwen35moe_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format qwen35moe GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "qwen35moe.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "qwen35moe.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "qwen35moe.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_u32_kv(meta, "qwen35moe.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "qwen35moe.vision.spatial_merge_size", "clip.vision.spatial_merge_size"); + copy_u32_kv(meta, "qwen35moe.vision.num_channels", "clip.vision.num_channels"); + // projection_dim = text model's embedding_length. + copy_u32_kv(meta, "qwen35moe.embedding_length", "clip.vision.projection_dim"); + + // Defaults for KVs Ollama omitted (match the Qwen3.5-35B-A3B reference mmproj). + inject_u32_if_missing(meta, "clip.vision.feed_forward_length", 4304); + inject_u32_if_missing(meta, "clip.vision.image_size", 768); + inject_f32_if_missing(meta, "clip.vision.attention.layer_norm_epsilon", 1e-6f); + + static const float kHalfHalfHalf[3] = {0.5f, 0.5f, 0.5f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kHalfHalfHalf, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kHalfHalfHalf, 3); + + // is_deepstack_layers: qwen3.5 35B has no deepstack layers. Set 27 False. + if (!has_key(meta, "clip.vision.is_deepstack_layers")) { + uint8_t bools[27] = {}; + gguf_set_arr_data(meta, "clip.vision.is_deepstack_layers", GGUF_TYPE_BOOL, bools, 27); + } + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_gelu", true); + gguf_set_val_str(meta, "clip.projector_type", "qwen3vl_merger"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // QKV merge runs BEFORE substring renames so it can find attn_q/k/v by name. + const int64_t n_blocks_key = gguf_find_key(meta, "clip.vision.block_count"); + const uint32_t n_blocks = n_blocks_key >= 0 ? gguf_get_val_u32(meta, n_blocks_key) : 27; + for (uint32_t b = 0; b < n_blocks; ++b) register_qwen35moe_qkv_merge(meta, ctx, (int) b); + + // Also before renames: patch_embed references the source by name. + register_qwen35moe_patch_embed_split(meta, ctx); + + // Simple substring renames. + for (const auto & [from, to] : kQwen35moeClipRenames) { + rename_tensors_containing(meta, ctx, from, to); + } + + promote_tensor_to_f32(meta, ctx, "v.position_embd.weight"); +} + +// ========================================================================= +// deepseek-ocr (clip side — SAM + CLIP + projector) +// ========================================================================= +// +// Ollama's monolithic deepseek-ocr GGUF embeds three vision components: +// * SAM encoder under the `s.*` prefix (12 blocks) +// * CLIP encoder under the `v.*` prefix (24 blocks) +// * MLP projector under `mm.*` +// Upstream's PROJECTOR_TYPE_DEEPSEEKOCR loader expects: +// * SAM under `v.sam.*` +// * CLIP under `v.*` (different leaf names than Ollama) +// * Projector as `mm.model.fc.*` plus `v.image_newline` / `v.view_seperator` + +constexpr std::pair kDeepseekocrClipRenames[] = { + // CLIP block leaf renames (also affects v.sam.* but those names don't overlap). + {".self_attn.out_proj", ".attn_out"}, + {".self_attn.qkv_proj", ".attn_qkv"}, + {".layer_norm1", ".ln1"}, + {".layer_norm2", ".ln2"}, + {".mlp.fc1", ".ffn_up"}, + {".mlp.fc2", ".ffn_down"}, + {"v.pre_layrnorm", "v.pre_ln"}, // Ollama typo + + // SAM block leaf renames (after `s.*` -> `v.sam.*` is applied). + {".attn.proj", ".attn.out"}, + {".attn.rel_pos_h", ".attn.pos_h.weight"}, + {".attn.rel_pos_w", ".attn.pos_w.weight"}, + {".norm1", ".pre_ln"}, + {".norm2", ".post_ln"}, + + // Projector renames. + {"mm.layers", "mm.model.fc"}, + {"mm.image_newline", "v.image_newline"}, + {"mm.view_seperator", "v.view_seperator"}, +}; + +void handle_deepseekocr_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format deepseekocr GGUF used as mmproj; translating\n", __func__); + + // CLIP encoder hparams. + copy_u32_kv(meta, "deepseekocr.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "deepseekocr.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "deepseekocr.vision.head_count", "clip.vision.attention.head_count"); + copy_u32_kv(meta, "deepseekocr.vision.image_size", "clip.vision.image_size"); + copy_u32_kv(meta, "deepseekocr.vision.patch_size", "clip.vision.patch_size"); + + // SAM encoder hparams. + copy_u32_kv(meta, "deepseekocr.sam.block_count", "clip.vision.sam.block_count"); + copy_u32_kv(meta, "deepseekocr.sam.embedding_length", "clip.vision.sam.embedding_length"); + copy_u32_kv(meta, "deepseekocr.sam.head_count", "clip.vision.sam.head_count"); + + // Defaults pulled from the upstream-converted reference mmproj. + inject_u32_if_missing(meta, "clip.vision.feed_forward_length", 64); + inject_u32_if_missing(meta, "clip.vision.projection_dim", 1280); + inject_u32_if_missing(meta, "clip.vision.projector.scale_factor", 1); + inject_u32_if_missing(meta, "clip.vision.window_size", 14); + inject_f32_if_missing(meta, "clip.vision.attention.layer_norm_epsilon", 1e-6f); + + static const float kHalfHalfHalf[3] = {0.5f, 0.5f, 0.5f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kHalfHalfHalf, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kHalfHalfHalf, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_gelu", true); + gguf_set_val_str(meta, "clip.projector_type", "deepseekocr"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Step 1: rename SAM prefix `s.` -> `v.sam.` only at the start of names + // (substring rename would corrupt e.g. `mm.layers.weight` -> `mm.layerv.sam.weight`). + { + std::vector sam_names; + const int64_t n = gguf_get_n_tensors(meta); + for (int64_t i = 0; i < n; ++i) { + std::string name(gguf_get_tensor_name(meta, i)); + if (name.size() >= 2 && name[0] == 's' && name[1] == '.') { + sam_names.push_back(std::move(name)); + } + } + for (const auto & old_name : sam_names) { + rename_tensor(meta, ctx, old_name.c_str(), + ("v.sam." + old_name.substr(2)).c_str()); + } + } + + // Step 2: SAM `s.position_embd` (no `.weight` suffix) — handle exactly, + // since after the `s.` rename it lives at `v.sam.position_embd`. + rename_tensor(meta, ctx, "v.sam.position_embd", "v.sam.pos_embd.weight"); + + // Step 3: substring renames for CLIP, SAM block leaves, and projector. + for (const auto & [from, to] : kDeepseekocrClipRenames) { + rename_tensors_containing(meta, ctx, from, to); + } + + // Metal IM2COL needs F32 patch_embd (same issue as gemma3 / mistral3). + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); + promote_tensor_to_f32(meta, ctx, "v.sam.patch_embd.weight"); + // CLIP position embedding too — Ollama stores F16, upstream stores F32. + promote_tensor_to_f32(meta, ctx, "v.position_embd.weight"); +} + +// ========================================================================= +// gemma4 (clip side — gemma4v projector) +// ========================================================================= +// +// Ollama's monolithic gemma4 GGUF embeds a SigLIP-style ViT plus the +// gemma4v projector (a single `mm.input_projection`). All v.* / mm.* +// tensor names already match upstream's PROJECTOR_TYPE_GEMMA4V — this +// handler only needs KV translation and an F32 promote of the patch +// embedding (Metal IM2COL). +// +// gemma4 vision uses image normalization mean=[0,0,0] / std=[1,1,1] +// (the LM does its own per-image normalization via v.std_bias / +// v.std_scale tensors) — different from the [0.5,0.5,0.5] used by +// most other arches. + +void handle_gemma4_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format gemma4 GGUF used as mmproj; translating\n", __func__); + + gguf_set_val_str(meta, "general.architecture", "clip"); + + const bool has_vision = any_tensor_with_prefix(ctx, "v."); + const bool has_audio = any_tensor_with_prefix(ctx, "a."); + + if (has_vision) { + copy_u32_kv(meta, "gemma4.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "gemma4.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "gemma4.vision.feed_forward_length", "clip.vision.feed_forward_length"); + copy_u32_kv(meta, "gemma4.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_f32_kv(meta, "gemma4.vision.attention.layer_norm_epsilon", "clip.vision.attention.layer_norm_epsilon"); + copy_u32_kv(meta, "gemma4.vision.patch_size", "clip.vision.patch_size"); + // gemma4 vision is fixed at 224x224 patches. + inject_u32_if_missing(meta, "clip.vision.image_size", 224); + // projection_dim = LM embedding length. + copy_u32_kv(meta, "gemma4.embedding_length", "clip.vision.projection_dim"); + + static const float kZeros[3] = {0.0f, 0.0f, 0.0f}; + static const float kOnes [3] = {1.0f, 1.0f, 1.0f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kZeros, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kOnes, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + gguf_set_val_str(meta, "clip.vision.projector_type", "gemma4v"); + + // Metal IM2COL needs F32 patch_embd weights (same as other arches). + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); + } + + if (has_audio) { + // Audio (gemma4a — conformer encoder + audio multimodal embedder). + copy_u32_kv(meta, "gemma4.audio.block_count", "clip.audio.block_count"); + copy_u32_kv(meta, "gemma4.audio.embedding_length", "clip.audio.embedding_length"); + copy_u32_kv(meta, "gemma4.audio.feed_forward_length", "clip.audio.feed_forward_length"); + copy_u32_kv(meta, "gemma4.audio.attention.head_count", "clip.audio.attention.head_count"); + copy_f32_kv(meta, "gemma4.audio.attention.layer_norm_epsilon", "clip.audio.attention.layer_norm_epsilon"); + // Defaults from the upstream-converted reference E2B mmproj. + inject_u32_if_missing(meta, "clip.audio.num_mel_bins", 128); + inject_u32_if_missing(meta, "clip.audio.projection_dim", 1536); + + inject_bool_if_missing(meta, "clip.has_audio_encoder", true); + gguf_set_val_str(meta, "clip.audio.projector_type", "gemma4a"); + + // Top-level tensor renames. Ollama uses different leaf names for the + // SSCP input projection and the encoder output projection: + // a.pre_encode.out.weight → a.input_projection.weight (SSCP proj) + // mm.a.fc.{weight,bias} → a.pre_encode.out.{weight,bias} + // mm.a.input_projection.weight already matches. + rename_tensor(meta, ctx, "a.pre_encode.out.weight", "a.input_projection.weight"); + rename_tensor(meta, ctx, "mm.a.fc.weight", "a.pre_encode.out.weight"); + rename_tensor(meta, ctx, "mm.a.fc.bias", "a.pre_encode.out.bias"); + + // Per-block renames. Scoped to a.blk.* (NOT vision blocks, which also + // have ln1/ln2). Order matters: ln2 → attn_post_norm must run before + // layer_pre_norm → ln2 (otherwise the second rename collides). + // + // Semantic mapping (from Ollama's model_audio.go vs upstream gemma4a.cpp): + // ln1 → attn_pre_norm (pre-attention norm) + // ln2 → attn_post_norm (post-attention norm; NOT block out) + // layer_pre_norm → ln2 (final block output norm) + // linear_pos → attn_k_rel (relative-position K projection) + const int kid = gguf_find_key(meta, "gemma4.audio.block_count"); + const uint32_t n_audio = (kid >= 0) ? gguf_get_val_u32(meta, kid) : 12; + for (uint32_t il = 0; il < n_audio; ++il) { + char from[GGML_MAX_NAME], to[GGML_MAX_NAME]; + auto rn = [&](const char * a, const char * b) { + std::snprintf(from, sizeof(from), "a.blk.%u.%s.weight", il, a); + std::snprintf(to, sizeof(to), "a.blk.%u.%s.weight", il, b); + rename_tensor(meta, ctx, from, to); + }; + rn("ln1", "attn_pre_norm"); + rn("ln2", "attn_post_norm"); + rn("layer_pre_norm", "ln2"); + rn("linear_pos", "attn_k_rel"); + } + } +} + +// ========================================================================= +// glm-ocr (clip side — glm4v projector) +// ========================================================================= +// +// Ollama stores the GLM4V vision tower with v.blk.X.* tensor names that +// already match upstream's expectations (`attn_qkv`, `attn_out`, +// `attn_q_norm`, `attn_k_norm`, `ln1`/`ln2`, `ffn_{gate,up,down}`). +// Most of mm.* (mm.model.fc, mm.up/gate/down, mm.post_norm, +// mm.patch_merger) is also already named correctly. The two diffs: +// * `v.patch_embd_0.weight` / `v.patch_embd_1.weight` → upstream's +// pixel-shuffle patch-embed pair `v.patch_embd.weight` / +// `v.patch_embd.weight.1`. +// * F32 promote of patch_embd weights (Metal IM2COL). + +void handle_glmocr_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format glm-ocr GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "glmocr.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "glmocr.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "glmocr.vision.intermediate_size", "clip.vision.feed_forward_length"); + copy_u32_kv(meta, "glmocr.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_f32_kv(meta, "glmocr.vision.attention.layer_norm_rms_epsilon", "clip.vision.attention.layer_norm_epsilon"); + copy_u32_kv(meta, "glmocr.vision.image_size", "clip.vision.image_size"); + copy_u32_kv(meta, "glmocr.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "glmocr.vision.spatial_merge_size", "clip.vision.spatial_merge_size"); + copy_u32_kv(meta, "glmocr.vision.out_hidden_size", "clip.vision.projection_dim"); + + // Ollama already shipped image_mean / image_std under glmocr.vision.*; + // copy them through. + { + const int64_t kid = gguf_find_key(meta, "glmocr.vision.image_mean"); + if (kid >= 0 && !has_key(meta, "clip.vision.image_mean")) { + const size_t n = gguf_get_arr_n(meta, kid); + gguf_set_arr_data(meta, "clip.vision.image_mean", GGUF_TYPE_FLOAT32, + gguf_get_arr_data(meta, kid), n); + } + } + { + const int64_t kid = gguf_find_key(meta, "glmocr.vision.image_std"); + if (kid >= 0 && !has_key(meta, "clip.vision.image_std")) { + const size_t n = gguf_get_arr_n(meta, kid); + gguf_set_arr_data(meta, "clip.vision.image_std", GGUF_TYPE_FLOAT32, + gguf_get_arr_data(meta, kid), n); + } + } + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_silu", true); + gguf_set_val_str(meta, "clip.projector_type", "glm4v"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Patch-embed temporal pair: Ollama uses _0/_1 suffixes, upstream uses + // unsuffixed/.1. + rename_tensor(meta, ctx, "v.patch_embd_0.weight", "v.patch_embd.weight"); + rename_tensor(meta, ctx, "v.patch_embd_1.weight", "v.patch_embd.weight.1"); + + // F32 promote for IM2COL on Metal (same fix as gemma3 / mistral3). + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight.1"); +} + +// ========================================================================= +// llama4 (clip side) +// ========================================================================= +// +// Ollama's monolithic llama4 GGUF embeds the CLIP-style ViT and a 3-layer +// projector (`mm.linear_1` + `v.vision_adapter.mlp.fc1/fc2`). Upstream's +// PROJECTOR_TYPE_LLAMA4 expects the projector under `mm.model.fc` / +// `mm.model.mlp.{1,2}` and standard CLIP block leaf names. + +constexpr std::pair kLlama4ClipRenames[] = { + // Vision-adapter MLP -> upstream's MM-MLP slots. Run BEFORE the generic + // `.mlp.fc{1,2}` -> `.ffn_{up,down}` rename so the substring match stays + // pinned to the adapter prefix. + {"v.vision_adapter.mlp.fc1", "mm.model.mlp.1"}, + {"v.vision_adapter.mlp.fc2", "mm.model.mlp.2"}, + + // Main projector. + {"mm.linear_1", "mm.model.fc"}, + + // Vision tower non-blk. + {"v.class_embedding", "v.class_embd"}, + {"v.layernorm_post", "v.post_ln"}, + {"v.layernorm_pre", "v.pre_ln"}, + {"v.patch_embedding", "v.patch_embd"}, + + // Vision-tower block leaves. + {".attn_output", ".attn_out"}, + {".attn_norm", ".ln1"}, + {".ffn_norm", ".ln2"}, + {".mlp.fc1", ".ffn_up"}, + {".mlp.fc2", ".ffn_down"}, +}; + +void handle_llama4_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format llama4 GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "llama4.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "llama4.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "llama4.vision.feed_forward_length", "clip.vision.feed_forward_length"); + copy_u32_kv(meta, "llama4.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_u32_kv(meta, "llama4.vision.image_size", "clip.vision.image_size"); + copy_u32_kv(meta, "llama4.vision.patch_size", "clip.vision.patch_size"); + copy_f32_kv(meta, "llama4.vision.layer_norm_epsilon", "clip.vision.attention.layer_norm_epsilon"); + // projection_dim = LM embedding length (= mm.model.fc output dim). + copy_u32_kv(meta, "llama4.embedding_length", "clip.vision.projection_dim"); + + // Defaults (match the upstream-converted reference mmproj). + inject_u32_if_missing(meta, "clip.vision.projector.scale_factor", 2); + + static const float kHalfHalfHalf[3] = {0.5f, 0.5f, 0.5f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kHalfHalfHalf, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kHalfHalfHalf, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_gelu", true); + gguf_set_val_str(meta, "clip.projector_type", "llama4"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Position embedding has no `.weight` suffix in Ollama; rename exactly. + rename_tensor(meta, ctx, "v.positional_embedding_vlm", "v.position_embd.weight"); + + for (const auto & [from, to] : kLlama4ClipRenames) { + rename_tensors_containing(meta, ctx, from, to); + } +} + +// ========================================================================= +// mistral3 (clip side — pixtral projector) +// ========================================================================= +// +// Tensor renames Ollama → upstream pixtral: +// v.patch_conv -> v.patch_embd +// v.encoder_norm -> v.pre_ln +// v.blk.X.attn_output -> v.blk.X.attn_out +// v.blk.X.attn_norm -> v.blk.X.ln1 +// v.blk.X.ffn_norm -> v.blk.X.ln2 +// mm.linear_1 -> mm.1 +// mm.linear_2 -> mm.2 +// mm.norm -> mm.input_norm +// mm.patch_merger.merging_layer -> mm.patch_merger +// +// img_break: pixtral's loader requires `v.token_embd.img_break` (the +// embedding row for the [IMG_BREAK] token, used as a row separator). +// Ollama's monolithic blob doesn't ship it as a separate tensor; the +// "ideal" value is row 12 of token_embd.weight, but token_embd is +// quantized (Q4_K) and per-row dequant is heavyweight. Reclaim the +// orphan output_norm.weight slot (already [n_embd] F32) and zero-fill +// it — pixtral.cpp adds img_break to row separator embeddings, so a +// zero embedding makes [IMG_BREAK] insertion a no-op without breaking +// the rest of the vision graph. +constexpr std::pair kMistral3ClipRenames[] = { + {"v.patch_conv", "v.patch_embd"}, + {"v.encoder_norm", "v.pre_ln"}, + {".attn_output", ".attn_out"}, + {".attn_norm", ".ln1"}, + {".ffn_norm", ".ln2"}, + {"mm.linear_1", "mm.1"}, + {"mm.linear_2", "mm.2"}, + {"mm.patch_merger.merging_layer", "mm.patch_merger"}, + {"mm.norm", "mm.input_norm"}, +}; + +// Apply the LLaMA-style RoPE permutation to Ollama's vision Q/K weight. +// +// Ollama's mistral3 converter (convert/convert_mistral.go) only applies +// its repack to TEXT-side attn_q/attn_k (the `if !HasPrefix(name, "v.")` +// guard skips vision tensors). So vision Q/K leave the converter in raw +// HF/PyTorch order. Upstream's HF→GGUF flow (convert_hf_to_gguf.py +// Mistral3 path) DOES permute vision Q/K with the vision head count, +// because pixtral's clip graph uses `ggml_rope_ext` in mode 0 which +// expects the [n_head, head_dim/2, 2, ...] layout. +// +// To bridge the two: apply LlamaModel.permute equivalently — reshape +// to [n_head, 2, head_dim/2, in], swap axes 1↔2, reshape back. The +// permutation acts only on the output dim, which is ne[1] for ggml +// weights stored as [in_dim, out_dim], so we shuffle whole rows. +// +// Permutation formula: oa = h*head_dim + dp*2 + half (post-permute idx) +// ob = h*head_dim + half*(head_dim/2) + dp (HF idx) +// copy row ob in src → row oa in dst. +// +// Only F16 Q/K rows handled (V is not RoPE'd; quantized rows would need +// block-aware shuffling — Ollama keeps Q/K F16 for mistral3 8B). +void register_mistral3_vision_qk_permute(gguf_context * meta, ggml_context * ctx, + const char * tensor_name, int n_head) { + ggml_tensor * t = ggml_get_tensor(ctx, tensor_name); + if (!t || t->type != GGML_TYPE_F16) return; + + const int total_out = (int) t->ne[1]; + if (total_out % n_head != 0) return; + const size_t row_bytes = ggml_row_size(t->type, t->ne[0]); + const size_t total_bytes = ggml_nbytes(t); + const size_t src_offset = tensor_file_offset(meta, tensor_name); + + const int head_dim = total_out / n_head; + const int head_dim2 = head_dim / 2; + + register_load_op(tensor_name, LoadOp{ + [=](const char * path, void * dst, size_t dst_size) { + if (dst_size != total_bytes) return false; + std::vector src(total_bytes); + if (!read_at(path, src_offset, src.data(), total_bytes)) return false; + uint8_t * dp = static_cast(dst); + for (int oa = 0; oa < total_out; ++oa) { + const int h = oa / head_dim; + const int dp_ = (oa % head_dim) / 2; + const int hf = oa % 2; + const int ob = h * head_dim + hf * head_dim2 + dp_; + std::memcpy(dp + (size_t) oa * row_bytes, + src.data() + (size_t) ob * row_bytes, row_bytes); + } + return true; + }, + "vision Q/K LLaMA permute", + }); +} + +void handle_mistral3_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format mistral3 GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "mistral3.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "mistral3.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "mistral3.vision.feed_forward_length", "clip.vision.feed_forward_length"); + copy_u32_kv(meta, "mistral3.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_u32_kv(meta, "mistral3.vision.image_size", "clip.vision.image_size"); + copy_u32_kv(meta, "mistral3.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "mistral3.vision.num_channels", "clip.vision.num_channels"); + copy_u32_kv(meta, "mistral3.spatial_merge_size", "clip.vision.spatial_merge_size"); + copy_f32_kv(meta, "mistral3.vision.rope.freq_base", "clip.rope.freq_base"); + // projection_dim is required by the loader but pixtral derives the + // actual output dim from mm_2_w shape — any non-zero value works. + // Mirror the LM embedding length for diagnostics-friendliness. + copy_u32_kv(meta, "mistral3.embedding_length", "clip.vision.projection_dim"); + + inject_f32_if_missing(meta, "clip.vision.attention.layer_norm_epsilon", 1e-5f); + + // Pixtral image stats (CLIP-style means). + static const float kPixtralMean[3] = {0.48145467f, 0.45782750f, 0.40821072f}; + static const float kPixtralStd [3] = {0.26862955f, 0.26130259f, 0.27577710f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kPixtralMean, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kPixtralStd, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_silu", true); + gguf_set_val_str(meta, "clip.projector_type", "pixtral"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Reclaim output_norm.weight as v.token_embd.img_break (zero-filled). + const int64_t lm_embd_kid = gguf_find_key(meta, "mistral3.embedding_length"); + const uint32_t lm_embd = lm_embd_kid >= 0 ? gguf_get_val_u32(meta, lm_embd_kid) : 0; + if (lm_embd > 0 && reclaim_slot_as(meta, ctx, + "output_norm.weight", "v.token_embd.img_break", + {(int64_t) lm_embd}, GGML_TYPE_F32)) { + register_load_op("v.token_embd.img_break", LoadOp{ + [](const char *, void * dst, size_t dst_size) { + std::memset(dst, 0, dst_size); + return true; + }, + "img_break zero-fill", + }); + } + + // Apply LLaMA-style RoPE permutation to vision Q/K BEFORE renames + // (we capture offsets by current name). Ollama's converter only + // repacks TEXT-side q/k (skipping `v.*`), but pixtral's clip graph + // expects HF→GGUF's permuted layout for vision Q/K. + { + const int64_t v_hk = gguf_find_key(meta, "mistral3.vision.attention.head_count"); + const int64_t n_blk_k = gguf_find_key(meta, "mistral3.vision.block_count"); + if (v_hk >= 0 && n_blk_k >= 0) { + const int n_head = (int) gguf_get_val_u32(meta, v_hk); + const uint32_t n_blocks = gguf_get_val_u32(meta, n_blk_k); + for (uint32_t b = 0; b < n_blocks; ++b) { + char qn[64], kn[64]; + std::snprintf(qn, sizeof(qn), "v.blk.%u.attn_q.weight", b); + std::snprintf(kn, sizeof(kn), "v.blk.%u.attn_k.weight", b); + register_mistral3_vision_qk_permute(meta, ctx, qn, n_head); + register_mistral3_vision_qk_permute(meta, ctx, kn, n_head); + } + } + } + + for (const auto & [from, to] : kMistral3ClipRenames) { + rename_tensors_containing(meta, ctx, from, to); + } + + // Upstream stores patch_embd as F32; Ollama stored F16. Metal's + // IM2COL convolution silently produces garbage with F16 weights + // (same issue as gemma3 — see handle_gemma3_clip). Promote to F32. + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); +} + +// ========================================================================= +// qwen25vl (clip side — Qwen2.5-VL vision tower + merger) +// ========================================================================= +// +// Ollama qwen25vl has a vision tower with mostly upstream-compatible +// tensor names. Five tensor renames + KV translation: +// +// v.merger.ln_q.weight → v.post_ln.weight (post-tower norm) +// v.merger.mlp.0.{weight,bias} → mm.0.{weight,bias} (LLaVA proj 0) +// v.merger.mlp.2.{weight,bias} → mm.2.{weight,bias} (LLaVA proj 2) +// v.patch_embd_0.weight → v.patch_embd.weight (slice 0) +// v.patch_embd_1.weight → v.patch_embd.weight.1 (slice 1) +// +// The KV side maps qwen25vl.vision.* → clip.vision.*, sets the projector +// type and use_silu, derives n_wa_pattern from fullatt_block_indexes[0]+1 +// (per upstream's qwen2.5vl converter), and supplies image_size=560 and +// projection_dim (= text embedding_length, qwen25vl.embedding_length). + +void handle_qwen25vl_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format qwen25vl GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "qwen25vl.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_f32_kv(meta, "qwen25vl.vision.attention.layer_norm_epsilon", "clip.vision.attention.layer_norm_epsilon"); + copy_u32_kv(meta, "qwen25vl.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "qwen25vl.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "qwen25vl.vision.num_channels", "clip.vision.num_channels"); + copy_u32_kv(meta, "qwen25vl.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "qwen25vl.vision.spatial_merge_size", "clip.vision.spatial_merge_size"); + copy_u32_kv(meta, "qwen25vl.vision.window_size", "clip.vision.window_size"); + copy_u32_kv(meta, "qwen25vl.embedding_length", "clip.vision.projection_dim"); + + // Derive feed_forward_length from the actual ffn_up shape if missing. + if (!has_key(meta, "clip.vision.feed_forward_length")) { + if (ggml_tensor * t = ggml_get_tensor(ctx, "v.blk.0.ffn_up.weight")) { + gguf_set_val_u32(meta, "clip.vision.feed_forward_length", (uint32_t) t->ne[1]); + } + } + + // Derive n_wa_pattern from fullatt_block_indexes[0]+1 (upstream convention). + { + const int64_t kid = gguf_find_key(meta, "qwen25vl.vision.fullatt_block_indexes"); + if (kid >= 0 && gguf_get_arr_n(meta, kid) >= 1) { + const auto * arr = static_cast(gguf_get_arr_data(meta, kid)); + gguf_set_val_u32(meta, "clip.vision.n_wa_pattern", (uint32_t)(arr[0] + 1)); + } + } + + // Default image_size = 560 (Qwen2VLVisionModel default, no image_size in HF config). + inject_u32_if_missing(meta, "clip.vision.image_size", 560); + + // Standard preprocessor mean/std for Qwen2.5-VL (CLIP convention). + static const float kMean[3] = {0.48145466f, 0.4578275f, 0.40821073f}; + static const float kStd [3] = {0.26862954f, 0.26130258f, 0.27577711f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kMean, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kStd, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_silu", true); + gguf_set_val_str(meta, "clip.projector_type", "qwen2.5vl_merger"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Tensor renames. + rename_tensor(meta, ctx, "v.merger.ln_q.weight", "v.post_ln.weight"); + rename_tensor(meta, ctx, "v.merger.mlp.0.weight", "mm.0.weight"); + rename_tensor(meta, ctx, "v.merger.mlp.0.bias", "mm.0.bias"); + rename_tensor(meta, ctx, "v.merger.mlp.2.weight", "mm.2.weight"); + rename_tensor(meta, ctx, "v.merger.mlp.2.bias", "mm.2.bias"); + rename_tensor(meta, ctx, "v.patch_embd_0.weight", "v.patch_embd.weight"); + rename_tensor(meta, ctx, "v.patch_embd_1.weight", "v.patch_embd.weight.1"); + + // Metal IM2COL needs F32 patch_embd (same issue as gemma3 / glmocr). + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight"); + promote_tensor_to_f32(meta, ctx, "v.patch_embd.weight.1"); +} + +// ========================================================================= +// qwen3vl (clip side — Qwen3-VL vision tower + deepstack adapters) +// ========================================================================= +// +// Ollama qwen3vl monolithic GGUF embeds the vision tower (27 blocks), +// deepstack merger adapters (3 of them, indexed 0/1/2), and the merger +// MLP. Compared to upstream's qwen3vl_merger expectations: +// +// * Per-block leaf renames: norm1→ln1, norm2→ln2, mlp.linear_fc1→ffn_up, +// mlp.linear_fc2→ffn_down. +// * Merger renames: v.merger.norm→v.post_ln, v.merger.linear_fc1→mm.0, +// v.merger.linear_fc2→mm.2 (LLaVA proj). +// * Deepstack remap: v.deepstack_merger.X.* → v.deepstack.{indexes[X]}.* +// where indexes is qwen3vl.vision.deepstack_visual_indexes (e.g. +// [8, 16, 24] for Qwen3-VL-8B). The leaf names also rename: +// linear_fc1→fc1, linear_fc2→fc2. +// * Per-block QKV merge: upstream's qwen3vl graph reads a single +// attn_qkv tensor (shape [hidden, 3*hidden]); Ollama stores separate +// Q/K/V. Same merge as qwen35moe — reuse that helper. +// * Patch embed: split the merged Conv3D weight [W,H,T,OUT*IN] into two +// Conv2D weights [W,H,IN,OUT], one per temporal slice. Same logic and +// donor (orphaned attn_k from QKV merge) as qwen35moe; reuse that helper. + +void handle_qwen3vl_clip(gguf_context * meta, ggml_context * ctx) { + LLAMA_LOG_INFO("%s: detected Ollama-format qwen3vl GGUF used as mmproj; translating\n", __func__); + + copy_u32_kv(meta, "qwen3vl.vision.attention.head_count", "clip.vision.attention.head_count"); + copy_f32_kv(meta, "qwen3vl.vision.attention.layer_norm_epsilon", "clip.vision.attention.layer_norm_epsilon"); + copy_u32_kv(meta, "qwen3vl.vision.block_count", "clip.vision.block_count"); + copy_u32_kv(meta, "qwen3vl.vision.embedding_length", "clip.vision.embedding_length"); + copy_u32_kv(meta, "qwen3vl.vision.num_channels", "clip.vision.num_channels"); + copy_u32_kv(meta, "qwen3vl.vision.patch_size", "clip.vision.patch_size"); + copy_u32_kv(meta, "qwen3vl.vision.spatial_merge_size", "clip.vision.spatial_merge_size"); + copy_u32_kv(meta, "qwen3vl.embedding_length", "clip.vision.projection_dim"); + + // Derive feed_forward_length from ffn_up / mlp.linear_fc1 shape. + if (!has_key(meta, "clip.vision.feed_forward_length")) { + if (ggml_tensor * t = ggml_get_tensor(ctx, "v.blk.0.mlp.linear_fc1.weight")) { + gguf_set_val_u32(meta, "clip.vision.feed_forward_length", (uint32_t) t->ne[1]); + } + } + + // image_size = sqrt(num_position_embeddings) * patch_size. v.pos_embed + // shape is [n_embd, num_positions], so num_positions = ne[1]. + if (!has_key(meta, "clip.vision.image_size")) { + ggml_tensor * pe = ggml_get_tensor(ctx, "v.pos_embed.weight"); + const int64_t patch_kid = gguf_find_key(meta, "qwen3vl.vision.patch_size"); + if (pe && patch_kid >= 0) { + const uint32_t patch = gguf_get_val_u32(meta, patch_kid); + const uint32_t side = (uint32_t) std::sqrt((double) pe->ne[1]); + gguf_set_val_u32(meta, "clip.vision.image_size", side * patch); + } + } + + // Image mean/std (Qwen3-VL uses [0.5, 0.5, 0.5] for both, per HF config). + static const float kHalfHalfHalf[3] = {0.5f, 0.5f, 0.5f}; + inject_f32_arr_if_missing(meta, "clip.vision.image_mean", kHalfHalfHalf, 3); + inject_f32_arr_if_missing(meta, "clip.vision.image_std", kHalfHalfHalf, 3); + + inject_bool_if_missing(meta, "clip.has_vision_encoder", true); + inject_bool_if_missing(meta, "clip.use_gelu", true); + gguf_set_val_str(meta, "clip.projector_type", "qwen3vl_merger"); + gguf_set_val_str(meta, "general.architecture", "clip"); + + // Per-block QKV merge: upstream's qwen3vl_merger graph reads a single + // `v.blk.X.attn_qkv.weight` (shape [hidden, 3*hidden]) — Ollama stores + // separate Q/K/V. Unlike qwen35moe (where Q/K/V are uniformly F16), the + // qwen3vl Ollama blob can mix F16 (Q/K) with Q8_0 (V), so a raw byte + // concat fails. Dequantize all three to F32 and concat in F32 instead. + // After the merge, attn_k/attn_v become orphaned in the clip ctx, which + // the patch_embed split then reclaims for `v.patch_embd.weight.1`. + const int64_t n_blocks_key = gguf_find_key(meta, "clip.vision.block_count"); + const uint32_t n_blocks = n_blocks_key >= 0 ? gguf_get_val_u32(meta, n_blocks_key) : 27; + for (uint32_t b = 0; b < n_blocks; ++b) { + char q[64], k[64], v[64], qb[64], kb[64], vb[64], qkv_w[64], qkv_b[64]; + std::snprintf(q, sizeof(q), "v.blk.%u.attn_q.weight", b); + std::snprintf(k, sizeof(k), "v.blk.%u.attn_k.weight", b); + std::snprintf(v, sizeof(v), "v.blk.%u.attn_v.weight", b); + std::snprintf(qb, sizeof(qb), "v.blk.%u.attn_q.bias", b); + std::snprintf(kb, sizeof(kb), "v.blk.%u.attn_k.bias", b); + std::snprintf(vb, sizeof(vb), "v.blk.%u.attn_v.bias", b); + std::snprintf(qkv_w, sizeof(qkv_w), "v.blk.%u.attn_qkv.weight", b); + std::snprintf(qkv_b, sizeof(qkv_b), "v.blk.%u.attn_qkv.bias", b); + if (!ggml_get_tensor(ctx, q)) continue; + + register_concat_load_to_f32(meta, ctx, qkv_w, {q, k, v}); + register_concat_load_to_f32(meta, ctx, qkv_b, {qb, kb, vb}); + + rename_tensor(meta, ctx, q, qkv_w); + if (ggml_tensor * t = ggml_get_tensor(ctx, qkv_w)) { + set_tensor_shape(t, {t->ne[0], t->ne[1] * 3}); + set_tensor_type (t, GGML_TYPE_F32); + } + rename_tensor(meta, ctx, qb, qkv_b); + if (ggml_tensor * t = ggml_get_tensor(ctx, qkv_b)) { + set_tensor_shape(t, {t->ne[0] * 3}); + set_tensor_type (t, GGML_TYPE_F32); + } + } + + // Patch embed split runs BEFORE per-block substring renames so it can + // find the source by name `v.patch_embed.weight`. Same shape as + // qwen35moe (16x16 patches, 2 temporal slices, 3 in_ch, 1152 out_ch). + register_qwen35moe_patch_embed_split(meta, ctx); + + // Top-level renames (full names) — must run before substring per-block + // renames so .linear_fc1 substring matches only inside .mlp.linear_fc1. + rename_tensor(meta, ctx, "v.merger.norm.weight", "v.post_ln.weight"); + rename_tensor(meta, ctx, "v.merger.norm.bias", "v.post_ln.bias"); + rename_tensor(meta, ctx, "v.merger.linear_fc1.weight", "mm.0.weight"); + rename_tensor(meta, ctx, "v.merger.linear_fc1.bias", "mm.0.bias"); + rename_tensor(meta, ctx, "v.merger.linear_fc2.weight", "mm.2.weight"); + rename_tensor(meta, ctx, "v.merger.linear_fc2.bias", "mm.2.bias"); + rename_tensor(meta, ctx, "v.patch_embed.bias", "v.patch_embd.bias"); + rename_tensor(meta, ctx, "v.pos_embed.weight", "v.position_embd.weight"); + + // Deepstack remap: v.deepstack_merger.X.{norm,linear_fc1,linear_fc2}.{weight,bias} + // → v.deepstack.{deepstack_visual_indexes[X]}.{norm,fc1,fc2}.{weight,bias}. + // Upstream stores deepstack tensors at the absolute clip layer index + // (e.g. v.deepstack.8.* for the adapter that fires after layer 8). + { + const int64_t ds_kid = gguf_find_key(meta, "qwen3vl.vision.deepstack_visual_indexes"); + if (ds_kid >= 0) { + const size_t n = gguf_get_arr_n(meta, ds_kid); + const auto * idx = static_cast(gguf_get_arr_data(meta, ds_kid)); + for (size_t i = 0; i < n; ++i) { + char from[GGML_MAX_NAME], to[GGML_MAX_NAME]; + auto rn = [&](const char * leaf_from, const char * leaf_to, const char * suffix) { + std::snprintf(from, sizeof(from), "v.deepstack_merger.%zu.%s.%s", i, leaf_from, suffix); + std::snprintf(to, sizeof(to), "v.deepstack.%d.%s.%s", idx[i], leaf_to, suffix); + rename_tensor(meta, ctx, from, to); + }; + rn("norm", "norm", "weight"); + rn("norm", "norm", "bias"); + rn("linear_fc1", "fc1", "weight"); + rn("linear_fc1", "fc1", "bias"); + rn("linear_fc2", "fc2", "weight"); + rn("linear_fc2", "fc2", "bias"); + } + } + } + + // Per-block substring renames (safe — these substrings now only appear + // in v.blk.X.* paths after the top-level/deepstack renames above). + rename_tensors_containing(meta, ctx, ".norm1", ".ln1"); + rename_tensors_containing(meta, ctx, ".norm2", ".ln2"); + rename_tensors_containing(meta, ctx, ".mlp.linear_fc1", ".ffn_up"); + rename_tensors_containing(meta, ctx, ".mlp.linear_fc2", ".ffn_down"); + + // Position embed should be F32 (precision matters for resize_position_embeddings). + promote_tensor_to_f32(meta, ctx, "v.position_embd.weight"); +} + +} // anonymous namespace + +// ========================================================================= +// Public entry points +// ========================================================================= + +bool translate_metadata(const llama_model_loader * ml, + gguf_context * meta, + ggml_context * ctx, + std::string & arch_name, + const char * fname) { + if (!meta) return false; + { + std::lock_guard lk(g_loader_path_mutex); + g_loader_paths[ml] = fname ? fname : ""; + } + // embeddinggemma must run before gemma3: it switches arch_name to + // "gemma-embedding", which is what later checks (and the loader's KV + // prefix) need to see. + if (arch_name == "gemma3") handle_embeddinggemma(ml, meta, ctx, arch_name); + if (arch_name == "gemma3") handle_gemma3 (ml, meta, ctx); + if (arch_name == "gemma3n") handle_gemma3n (ml, meta, ctx); + if (arch_name == "gemma4") handle_gemma4 (ml, meta, ctx); + if (arch_name == "qwen35moe") handle_qwen35moe(ml, meta, ctx); + if (arch_name == "qwen35") handle_qwen35 (ml, meta, ctx); + if (arch_name == "gptoss") handle_gptoss (ml, meta, ctx, arch_name); + if (arch_name == "lfm2") handle_lfm2 (ml, meta, ctx); + if (arch_name == "mistral3") handle_mistral3 (ml, meta, ctx); + // qwen25vl must run before any qwen2vl-targeted handler — it switches + // arch_name to "qwen2vl" so the loader uses qwen2vl.* keys. + if (arch_name == "qwen25vl") handle_qwen25vl (ml, meta, ctx, arch_name); + if (arch_name == "qwen3vl") handle_qwen3vl (ml, meta, ctx); + // glm4moelite switches arch_name to "deepseek2" — same pattern. + if (arch_name == "glm4moelite") handle_glm4moelite (ml, meta, ctx, arch_name); + if (arch_name == "deepseekocr") handle_deepseekocr (ml, meta, ctx, arch_name); + if (arch_name == "nemotron_h_moe") handle_nemotron_h_moe(ml, meta, ctx); + if (arch_name == "llama4") handle_llama4 (ml, meta, ctx); + if (arch_name == "glmocr") handle_glmocr (ml, meta, ctx, arch_name); + // Dispatch. Add more arches as they are wired up. + + return is_mmap_disabled_for(ml); +} + +void translate_clip_metadata(gguf_context * meta, ggml_context * ctx) { + if (!meta) return; + if (!any_tensor_with_prefix(ctx, "v.")) return; // nothing to translate + + if (detect_ollama_gemma3(meta, ctx)) { + LLAMA_LOG_INFO("%s: detected Ollama-format gemma3 GGUF used as mmproj; translating\n", __func__); + handle_gemma3_clip(meta, ctx); + return; + } + if (detect_ollama_qwen35moe(meta, ctx)) { + handle_qwen35moe_clip(meta, ctx); + return; + } + if (detect_ollama_mistral3(meta, ctx)) { + handle_mistral3_clip(meta, ctx); + return; + } + if (detect_ollama_deepseekocr(meta)) { + handle_deepseekocr_clip(meta, ctx); + return; + } + if (detect_ollama_llama4(meta, ctx)) { + handle_llama4_clip(meta, ctx); + return; + } + if (detect_ollama_gemma4(meta, ctx)) { + handle_gemma4_clip(meta, ctx); + return; + } + if (detect_ollama_glmocr(meta)) { + handle_glmocr_clip(meta, ctx); + return; + } + if (detect_ollama_qwen25vl(meta)) { + handle_qwen25vl_clip(meta, ctx); + return; + } + if (detect_ollama_qwen3vl(meta, ctx)) { + handle_qwen3vl_clip(meta, ctx); + return; + } +} + +bool should_skip_tensor(const llama_model_loader * ml, const char * tensor_name) { + return should_skip_tensor_prefix(ml, tensor_name); +} + +bool maybe_load_tensor(ggml_tensor * cur, + const char * source_file, + size_t file_offset, + ggml_backend_buffer_type_t buft) { + (void) file_offset; // registered ops capture their own offsets + + LoadOp op; + if (!take_load_op(ggml_get_name(cur), op)) return false; + + const size_t dst_size = ggml_nbytes(cur); + std::vector dst(dst_size); + if (!op.apply(source_file, dst.data(), dst_size)) { + LLAMA_LOG_ERROR("%s: %s failed for %s\n", __func__, op.description, ggml_get_name(cur)); + return false; + } + + // buft can be null for tensors not yet bound to a backend buffer (e.g. + // tied output reusing token_embd's storage). In that case the tensor + // already has a host-side data pointer — write to it directly. + const bool is_host = !buft || ggml_backend_buft_is_host(buft); + if (is_host) { + if (!cur->data) { + LLAMA_LOG_ERROR("%s: no destination for %s (no buffer, no data)\n", __func__, ggml_get_name(cur)); + return false; + } + std::memcpy(cur->data, dst.data(), dst_size); + } else { + ggml_backend_tensor_set(cur, dst.data(), 0, dst_size); + } + + LLAMA_LOG_INFO("%s: %s for %s (%zu bytes)\n", __func__, op.description, ggml_get_name(cur), dst_size); + return true; +} + +bool maybe_load_text_tensor(const llama_model_loader * ml, + ggml_tensor * cur, + size_t file_offset) { + std::string path; + { + std::lock_guard lk(g_loader_path_mutex); + auto it = g_loader_paths.find(ml); + if (it == g_loader_paths.end() || it->second.empty()) return false; + path = it->second; + } + ggml_backend_buffer_type_t buft = cur->buffer + ? ggml_backend_buffer_get_type(cur->buffer) + : nullptr; + return maybe_load_tensor(cur, path.c_str(), file_offset, buft); +} + +} // namespace llama_ollama_compat diff --git a/llama/compat/llama-ollama-compat.h b/llama/compat/llama-ollama-compat.h new file mode 100644 index 000000000..3444571ea --- /dev/null +++ b/llama/compat/llama-ollama-compat.h @@ -0,0 +1,82 @@ +#pragma once + +// Ollama-format GGUF compatibility shim. +// +// Older Ollama builds ship GGUFs that differ from upstream in a handful of +// ways per-architecture (arch names, KV keys, tensor names, file layout). +// This shim detects those files during load and translates them in-memory +// so the rest of llama.cpp can load them unmodified. +// +// Three upstream hook points call into this namespace — one per insertion: +// +// 1. llama-model-loader.cpp (main model load): +// translate_metadata() — mutate KVs / tensor metadata +// should_skip_tensor() — filter weights_map population +// +// 2. tools/mtmd/clip.cpp (mmproj load): +// translate_clip_metadata() — rewrite KVs + tensor names for clip +// maybe_load_tensor() — override file read (e.g. F16->F32) +// +// Detection is per-arch; for any non-Ollama file every entry point is a +// no-op. Per-arch logic lives in anonymous-namespace handle_() +// functions in the .cpp; adding a new arch is a new handler plus one +// dispatch line in each translate_* entry point. + +#include +#include + +#include "ggml-backend.h" // for ggml_backend_buffer_type_t + +struct gguf_context; +struct ggml_context; +struct ggml_tensor; +struct llama_model_loader; + +namespace llama_ollama_compat { + +// Called from llama_model_loader's constructor, right after the arch is read. +// `fname` is the model file path, captured here so later load-time hooks +// (maybe_load_text_tensor) can read raw bytes from it. +// +// Returns true if the caller must disable mmap for this loader. Some +// handlers transform tensor data via load_op (e.g. glm-ocr's gate+up +// FFN concat), which is incompatible with the default mmap path: +// the upstream loader binds tensors directly to the mmap'd file region, +// so there's nowhere to write the transformed bytes. Disabling mmap +// makes the loader pre-allocate real backend buffers, after which our +// load_op overrides land in writable memory. +bool translate_metadata(const llama_model_loader * ml, + gguf_context * meta, + ggml_context * ctx, + std::string & arch_name, + const char * fname); + +// Called from llama_model_loader's weights_map population loop. Returns +// true to drop a tensor from the loader — used to hide embedded vision +// tensors from the text model's view without modifying the gguf_context. +bool should_skip_tensor(const llama_model_loader * ml, const char * tensor_name); + +// Called from clip_model_loader's constructor. Rewrites the clip-facing +// view of the metadata (arch=clip, clip.vision.* KVs, renamed tensors) +// so the rest of clip.cpp can load an Ollama monolithic GGUF unchanged. +void translate_clip_metadata(gguf_context * meta, ggml_context * ctx); + +// Called from clip.cpp's tensor-loading loop, before the normal file read. +// If this tensor was marked for type promotion by translate_clip_metadata +// (e.g. F16->F32), performs the conversion and writes the result into +// `cur` (host memcpy or backend_tensor_set based on `buft`). Returns true +// when the tensor was handled — caller should skip its normal read path. +bool maybe_load_tensor(ggml_tensor * cur, + const char * source_file, + size_t file_offset, + ggml_backend_buffer_type_t buft); + +// Text-side counterpart to maybe_load_tensor. Self-contained: looks up +// the model file path from the per-loader registry populated by +// translate_metadata, and derives the buffer type from cur->buffer +// internally — keeps the call site (and the upstream patch) to one line. +bool maybe_load_text_tensor(const llama_model_loader * ml, + ggml_tensor * cur, + size_t file_offset); + +} // namespace llama_ollama_compat diff --git a/llama/compat/upstream-edits.patch b/llama/compat/upstream-edits.patch new file mode 100644 index 000000000..76cc65048 --- /dev/null +++ b/llama/compat/upstream-edits.patch @@ -0,0 +1,78 @@ +diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp +index 4e65a45a5..75836c683 100644 +--- a/src/llama-model-loader.cpp ++++ b/src/llama-model-loader.cpp +@@ -4,6 +4,7 @@ + #include "ggml.h" + #include "gguf.h" + #include "llama-hparams.h" ++#include "llama-ollama-compat.h" + + #include + #include +@@ -549,6 +550,7 @@ llama_model_loader::llama_model_loader( + } + + get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); ++ if (llama_ollama_compat::translate_metadata(this, metadata, ctx, arch_name, fname.c_str())) use_mmap = false; + llm_kv = LLM_KV(llm_arch_from_string(arch_name)); + + files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io)); +@@ -573,6 +575,9 @@ llama_model_loader::llama_model_loader( + // so we build a unified tensors index for weights. + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + std::string tensor_name = std::string(cur->name); ++ if (llama_ollama_compat::should_skip_tensor(this, tensor_name.c_str())) { ++ continue; ++ } + // make sure there is no duplicated tensor names + if (weights_map.find(tensor_name) != weights_map.end()) { + throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); +@@ -683,6 +688,9 @@ llama_model_loader::llama_model_loader( + // Save tensors data offset info of the main file. + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + std::string tensor_name = std::string(cur->name); ++ if (llama_ollama_compat::should_skip_tensor(this, tensor_name.c_str())) { ++ continue; ++ } + // make sure there is no duplicated tensor names + if (weights_map.find(tensor_name) != weights_map.end()) { + throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); +@@ -1535,3 +1542,4 @@ bool llama_model_loader::load_all_data( + size_t n_size = ggml_nbytes(cur); + ++ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, weight->offs)) continue; + if (use_mmap) { +diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp +index f0e8786b6..35defa89d 100644 +--- a/tools/mtmd/clip.cpp ++++ b/tools/mtmd/clip.cpp +@@ -10,6 +10,8 @@ + #include "ggml-backend.h" + #include "gguf.h" + ++#include "llama-ollama-compat.h" ++ + #include + #include + #include +@@ -985,6 +987,11 @@ struct clip_model_loader { + + ctx_meta.reset(meta); + ++ // If this is an Ollama-format monolithic GGUF (text + embedded ++ // vision), translate its metadata and tensor names into the ++ // upstream mmproj shape so the rest of this loader runs unchanged. ++ llama_ollama_compat::translate_clip_metadata(ctx_gguf.get(), meta); ++ + const int n_tensors = gguf_get_n_tensors(ctx_gguf.get()); + + // print gguf info +@@ -2358,6 +2365,7 @@ struct clip_model_loader { + auto it_off = tensor_offset.find(t->name); + GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); + const size_t offset = it_off->second; ++ if (llama_ollama_compat::maybe_load_tensor(cur, fname.c_str(), offset, buft)) continue; + fin.seekg(offset, std::ios::beg); + if (!fin) { + throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name)); diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt index 2d5cf46c8..962b40381 100644 --- a/llama/server/CMakeLists.txt +++ b/llama/server/CMakeLists.txt @@ -35,6 +35,20 @@ if(DEFINED ENV{OLLAMA_LLAMA_CPP_SOURCE}) message(STATUS "Using local llama.cpp source: ${_src}") endif() +# Ollama-compat shim: overlays the fetched llama.cpp source with a tiny +# in-memory translation layer that lets upstream llama-server load GGUFs +# produced by older Ollama versions (e.g. existing ~/.ollama/models/blobs). +# See llama/compat/README.md for details. +# +# The patch only runs when fetching from GitHub — if a local source override +# is active, leave the developer's tree alone (they can apply by hand if +# they want to iterate on the compat layer). +set(_ollama_compat_patch_cmd "") +if(NOT DEFINED ENV{OLLAMA_LLAMA_CPP_SOURCE}) + include(${CMAKE_CURRENT_SOURCE_DIR}/../compat/compat.cmake) + set(_ollama_compat_patch_cmd PATCH_COMMAND ${OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND}) +endif() + # Configure upstream build options BEFORE FetchContent_MakeAvailable. # When included via FetchContent, llama.cpp sets LLAMA_STANDALONE=OFF # so all optional builds default to OFF. We explicitly enable what we need. @@ -53,9 +67,27 @@ FetchContent_Declare( GIT_REPOSITORY "https://github.com/ggml-org/llama.cpp.git" GIT_TAG ${LLAMA_CPP_GIT_TAG} GIT_SHALLOW TRUE + ${_ollama_compat_patch_cmd} ) FetchContent_MakeAvailable(llama_cpp) +# Link the Ollama-compat source files into the fetched llama target. +# Kept separate from the upstream-edits patch so our .cpp/.h stay +# on-disk in llama/compat/ rather than being copied into _deps/. +if(DEFINED OLLAMA_LLAMA_CPP_COMPAT_DIR) + file(GLOB _compat_sources CONFIGURE_DEPENDS + ${OLLAMA_LLAMA_CPP_COMPAT_DIR}/*.cpp) + target_sources(llama PRIVATE ${_compat_sources}) + target_include_directories(llama PRIVATE + ${OLLAMA_LLAMA_CPP_COMPAT_DIR}) + # mtmd's clip.cpp #include's the compat header too — add the same dir + # to its PRIVATE include path (PRIVATE on llama doesn't propagate). + if(TARGET mtmd) + target_include_directories(mtmd PRIVATE + ${OLLAMA_LLAMA_CPP_COMPAT_DIR}) + endif() +endif() + # Find GPU toolkits for runtime dependency bundling. # The upstream llama.cpp build finds these internally, but we need the # variables (CUDAToolkit_LIBRARY_DIR, etc.) in our install scope. diff --git a/llm/llama_server.go b/llm/llama_server.go index 3ebf853f3..bf866bbb2 100644 --- a/llm/llama_server.go +++ b/llm/llama_server.go @@ -423,6 +423,33 @@ func NewLlamaServerRunner( // Check if this is an embedding model _, isEmbedding := f.KV()[fmt.Sprintf("%s.pooling_type", f.KV().Architecture())] + // Older Ollama-format GGUFs store vision tensors (v.*, mm.*) inline in + // the main model file rather than in a separate projector layer. When + // the arch has a llama/compat clip handler, we can point --mmproj at + // the same file and the in-process shim translates the two views. + // + // If we auto-enable --mmproj for an arch whose clip handler doesn't + // exist yet, upstream's clip loader sees un-translated Ollama tensors + // and aborts model load. So gate on an explicit allowlist that mirrors + // the compat layer's clip-side coverage in llama/compat/. + compatClipArches := map[string]bool{ + "gemma3": true, + "gemma4": true, + "qwen35moe": true, + "qwen25vl": true, + "qwen3vl": true, + "mistral3": true, + "deepseekocr": true, + "glmocr": true, + "llama4": true, + // Add entries as llama/compat grows clip handlers. + } + if len(projectors) == 0 && + len(f.Tensors().Items("v.")) > 0 && + compatClipArches[f.KV().Architecture()] { + projectors = []string{modelPath} + } + gpuLibs := ml.LibraryPaths(gpus) status := NewStatusWriter(os.Stderr) diff --git a/server/model_resolver.go b/server/model_resolver.go index 4dab91167..cbbeffa37 100644 --- a/server/model_resolver.go +++ b/server/model_resolver.go @@ -1,73 +1,10 @@ package server import ( - "log/slog" - "strings" - "github.com/ollama/ollama/internal/modelref" "github.com/ollama/ollama/types/model" ) -// Temporary redirection logic to map incompatible library models to compatible versions -var compatModelRedirects = []struct{ from, to string }{ - {"library/gpt-oss", "dhiltgen/gpt-oss"}, - {"library/gemma3", "dhiltgen/gemma3"}, - {"library/embeddinggemma", "dhiltgen/embeddinggemma"}, - {"library/snowflake-arctic-embed2", "dhiltgen/snowflake-arctic-embed2"}, - {"library/gemma3n", "dhiltgen/gemma3n"}, - {"library/glm-4.7-flash", "dhiltgen/glm-4.7-flash"}, - {"library/deepseek-ocr", "dhiltgen/deepseek-ocr"}, - {"library/glm-ocr", "dhiltgen/glm-ocr"}, - {"library/gemma4", "dhiltgen/gemma4"}, - {"library/qwen2.5vl", "dhiltgen/qwen2.5vl"}, - {"library/qwen3-vl", "dhiltgen/qwen3-vl"}, -} - -// applyCompatRedirect checks if a model name matches a compat redirect and -// returns the redirected name. Returns the original name if no redirect applies. -func applyCompatRedirect(n model.Name) (model.Name, bool) { - if strings.Contains(n.DisplayShortest(), "-cloud") { - return n, false - } - - for _, r := range compatModelRedirects { - fromNS, fromModel, _ := strings.Cut(r.from, "/") - if fromNS == n.Namespace && fromModel == n.Model { - redirected := n - toNS, toRest, _ := strings.Cut(r.to, "/") - redirected.Namespace = toNS - // Support "namespace/model:tag" to override the tag - if toModel, toTag, hasTag := strings.Cut(toRest, ":"); hasTag { - redirected.Model = toModel - redirected.Tag = toTag - } else { - redirected.Model = toRest - } - slog.Debug("redirecting to compatible model", "from", n.DisplayShortest(), "to", redirected.DisplayShortest()) - return redirected, true - } - } - return n, false -} - -// reverseCompatRedirect maps a redirected name back to its original library name. -// Used by PsHandler so users see the name they requested, not the internal redirect target. -// TODO: consider removing this before merging — it papers over the fact that -// the scheduler stores the redirected name instead of the user-facing name. -func reverseCompatRedirect(n model.Name) model.Name { - for _, r := range compatModelRedirects { - toNS, toModel, _ := strings.Cut(r.to, "/") - if toNS == n.Namespace && toModel == n.Model { - fromNS, fromModel, _ := strings.Cut(r.from, "/") - reversed := n - reversed.Namespace = fromNS - reversed.Model = fromModel - return reversed - } - } - return n -} - type modelSource = modelref.ModelSource const ( diff --git a/server/routes.go b/server/routes.go index 7bbbe92f8..0440387ba 100644 --- a/server/routes.go +++ b/server/routes.go @@ -1084,9 +1084,6 @@ func getExistingName(n model.Name) (model.Name, error) { } } - // Redirect models that have been republished in a compatible format - n, _ = applyCompatRedirect(n) - return n, nil } @@ -2157,10 +2154,7 @@ func (s *Server) PsHandler(c *gin.Context) { for _, v := range s.sched.loaded { m := v.model - // Show the user-facing name (pre-redirect) so ps output matches - // what the user originally requested. - // TODO: consider removing before merging — see reverseCompatRedirect comment - displayName := reverseCompatRedirect(model.ParseName(m.ShortName)).DisplayShortest() + displayName := model.ParseName(m.ShortName).DisplayShortest() modelDetails := api.ModelDetails{ Format: m.Config.ModelFormat, Family: m.Config.ModelFamily,