diff --git a/cmake/local.cmake b/cmake/local.cmake index 65a46eede..94d88f72c 100644 --- a/cmake/local.cmake +++ b/cmake/local.cmake @@ -111,6 +111,17 @@ else() file(READ "${CMAKE_SOURCE_DIR}/LLAMA_CPP_VERSION" OLLAMA_LLAMA_CPP_GIT_TAG) string(STRIP "${OLLAMA_LLAMA_CPP_GIT_TAG}" OLLAMA_LLAMA_CPP_GIT_TAG) include(${CMAKE_SOURCE_DIR}/llama/compat/compat.cmake) + include(${CMAKE_SOURCE_DIR}/llama/models/models.cmake) + + # When pre-fetching the shared llama.cpp source, apply the compat hooks + # patch plus every new-architecture registration patch (llama/models/*.patch). + # The llama/server sub-build then runs with OLLAMA_LLAMA_CPP_SKIP_COMPAT_PATCH=ON + # and only links the Ollama-owned sources against this already-patched tree. + set(OLLAMA_LLAMA_CPP_PATCH_COMMAND + ${CMAKE_COMMAND} + -DPATCH_FILE=${OLLAMA_LLAMA_CPP_COMPAT_PATCH_FILE} + -DPATCH_DIR=${OLLAMA_LLAMA_CPP_MODELS_DIR} + -P ${OLLAMA_LLAMA_CPP_COMPAT_DIR}/apply-patch.cmake) if(DEFINED FETCHCONTENT_SOURCE_DIR_LLAMA_CPP AND NOT "${FETCHCONTENT_SOURCE_DIR_LLAMA_CPP}" STREQUAL "") get_filename_component(OLLAMA_LLAMA_CPP_SOURCE_DIR "${FETCHCONTENT_SOURCE_DIR_LLAMA_CPP}" ABSOLUTE BASE_DIR "${CMAKE_SOURCE_DIR}") @@ -131,7 +142,7 @@ else() CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" - PATCH_COMMAND ${OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND} + PATCH_COMMAND ${OLLAMA_LLAMA_CPP_PATCH_COMMAND} USES_TERMINAL_DOWNLOAD TRUE USES_TERMINAL_PATCH TRUE) endif() diff --git a/llama/compat/apply-patch.cmake b/llama/compat/apply-patch.cmake index a349dfb71..80cbcfdb7 100644 --- a/llama/compat/apply-patch.cmake +++ b/llama/compat/apply-patch.cmake @@ -1,18 +1,19 @@ -# Idempotent patch applier used by compat.cmake. +# Idempotent patch applier used by compat.cmake and the new-architecture +# framework in llama/models/. # # Invocation (from a CMake PATCH_COMMAND): -# cmake -DPATCH_FILE= -P apply-patch.cmake +# cmake -DPATCH_FILE= [-DPATCH_DIR=] -P apply-patch.cmake # -# The patch is applied in the current working directory (which ExternalProject -# / FetchContent sets to the fetched source's SOURCE_DIR). If the patch is -# already applied — detected via `git apply --reverse --check` — this script -# is a no-op. This makes re-configuring and re-building the project safe. +# Applies PATCH_FILE first (if set), then every /*.patch in sorted +# order (if set). At least one of PATCH_FILE / PATCH_DIR must be provided. +# +# Patches are applied in the current working directory, which FetchContent / +# ExternalProject sets to the fetched source's SOURCE_DIR. A patch that can be +# REVERSED cleanly is treated as already applied and skipped, so re-configuring +# or rebuilding is safe. -if(NOT DEFINED PATCH_FILE) - message(FATAL_ERROR "apply-patch.cmake: PATCH_FILE not set") -endif() -if(NOT EXISTS "${PATCH_FILE}") - message(FATAL_ERROR "apply-patch.cmake: PATCH_FILE does not exist: ${PATCH_FILE}") +if(NOT DEFINED PATCH_FILE AND NOT DEFINED PATCH_DIR) + message(FATAL_ERROR "apply-patch.cmake: set PATCH_FILE and/or PATCH_DIR") endif() find_package(Git QUIET REQUIRED) @@ -21,30 +22,48 @@ get_filename_component(_patch_workdir "." ABSOLUTE) get_filename_component(_git_ceiling "${_patch_workdir}" DIRECTORY) set(_git_apply_env GIT_CEILING_DIRECTORIES=${_git_ceiling}) -# If the patch can be REVERSED cleanly, it's already applied. Skip. -execute_process( - COMMAND ${CMAKE_COMMAND} -E env ${_git_apply_env} - ${GIT_EXECUTABLE} apply --reverse --check "${PATCH_FILE}" - RESULT_VARIABLE _reverse_check - OUTPUT_QUIET ERROR_QUIET -) -if(_reverse_check EQUAL 0) - message(STATUS "llama/compat: patch already applied, skipping") - return() +function(_ollama_apply_patch patch_file) + if(NOT EXISTS "${patch_file}") + message(FATAL_ERROR "apply-patch.cmake: patch does not exist: ${patch_file}") + endif() + + # If the patch can be REVERSED cleanly, it's already applied. Skip. + execute_process( + COMMAND ${CMAKE_COMMAND} -E env ${_git_apply_env} + ${GIT_EXECUTABLE} apply --reverse --check "${patch_file}" + RESULT_VARIABLE _reverse_check + OUTPUT_QUIET ERROR_QUIET + ) + if(_reverse_check EQUAL 0) + message(STATUS "llama patch: already applied, skipping ${patch_file}") + return() + endif() + + # Otherwise, apply forward. + execute_process( + COMMAND ${CMAKE_COMMAND} -E env ${_git_apply_env} + ${GIT_EXECUTABLE} apply --whitespace=nowarn "${patch_file}" + RESULT_VARIABLE _apply_result + ) + if(NOT _apply_result EQUAL 0) + message(FATAL_ERROR + "llama patch: failed to apply ${patch_file}\n" + "This usually means the pinned llama.cpp source has changed. " + "Regenerate the patch (see the matching README) against the " + "pinned LLAMA_CPP_VERSION and retry.") + endif() + + message(STATUS "llama patch: applied ${patch_file}") +endfunction() + +if(DEFINED PATCH_FILE AND NOT PATCH_FILE STREQUAL "") + _ollama_apply_patch("${PATCH_FILE}") endif() -# Otherwise, apply forward. -execute_process( - COMMAND ${CMAKE_COMMAND} -E env ${_git_apply_env} - ${GIT_EXECUTABLE} apply --whitespace=nowarn "${PATCH_FILE}" - RESULT_VARIABLE _apply_result -) -if(NOT _apply_result EQUAL 0) - message(FATAL_ERROR - "llama/compat: failed to apply ${PATCH_FILE}\n" - "This usually means the pinned llama.cpp source has changed. " - "Regenerate the patch (see llama/compat/README.md) against the " - "pinned LLAMA_CPP_VERSION and retry.") +if(DEFINED PATCH_DIR AND NOT PATCH_DIR STREQUAL "") + file(GLOB _ollama_patches "${PATCH_DIR}/*.patch") + list(SORT _ollama_patches) + foreach(_ollama_patch IN LISTS _ollama_patches) + _ollama_apply_patch("${_ollama_patch}") + endforeach() endif() - -message(STATUS "llama/compat: applied patch") diff --git a/llama/models/README.md b/llama/models/README.md new file mode 100644 index 000000000..7e87b58f4 --- /dev/null +++ b/llama/models/README.md @@ -0,0 +1,57 @@ +# Interim model-architecture support + +Ollama runs GGUF models with upstream llama.cpp, pinned at `LLAMA_CPP_VERSION`. +Occasionally a model arrives whose architecture isn't in that pinned version +yet. To keep that model working in the meantime, we carry a small, temporary +addition here that teaches the fetched llama.cpp about it, following llama.cpp's +own model conventions so the same work can be offered upstream. + +This is deliberately interim. As soon as the architecture is available in +llama.cpp, the files below are deleted and the model loads on stock llama.cpp. + +> Its counterpart, `llama/compat/`, handles the opposite case: models llama.cpp +> *already* supports, whose older GGUF files just need their metadata translated. + +## What's here, per architecture + +- `.cpp` — the model implementation (hparams, tensors, compute graph). It + lives in our tree and is compiled into llama.cpp via CMake, so a llama.cpp + version bump leaves it untouched. +- `llama-cpp-.patch` — the few edits that register the architecture in + llama.cpp's own tables: the arch name, the model factory, the rope type, and — + only if the model needs them — a tensor name or tokenizer entry. + +The build applies the patch and links the source automatically. + +## Adding one + +Work against a llama.cpp checkout at the pinned `LLAMA_CPP_VERSION`: + +1. Implement the architecture there as an ordinary llama.cpp model (a + `src/models/.cpp` plus the registration edits), modeled on the closest + existing architecture and reusing its building blocks. Iterate with + `llama-cli` until it loads and generates correctly. +2. Move the implementation to `llama/models/.cpp`, changing its include + from `"models.h"` to `"models/models.h"`. +3. Capture the registration edits as the patch — only the files you changed: + ```sh + git diff -- src/llama-arch.h src/llama-arch.cpp src/llama-model.cpp \ + src/models/models.h src/llama-vocab.h src/llama-vocab.cpp \ + > llama/models/llama-cpp-.patch + ``` +4. Build from the repo root (`cmake -B build . && cmake --build build`) and + confirm `ollama run ` works and stops cleanly. + +Keep the footprint small: put the logic in `.cpp`, and prefer reusing +llama.cpp's existing hparams, tensor names, and graph builders over adding new +ones. The smaller the patch, the less there is to redo on a version bump. + +## After a llama.cpp bump + +Re-apply the registration edits to the new checkout and re-capture the diff; the +`.cpp` usually needs no change. If the architecture has landed upstream by +then, simply delete `.cpp` and its patch. + +## Current architectures + +- `laguna` — poolside Laguna. Remove once upstream llama.cpp supports it. diff --git a/llama/models/laguna.cpp b/llama/models/laguna.cpp new file mode 100644 index 000000000..f32b107c6 --- /dev/null +++ b/llama/models/laguna.cpp @@ -0,0 +1,268 @@ +// Laguna (poolside) architecture implementation for llama.cpp. +// +// This file lives in Ollama's tree and is compiled into the fetched llama.cpp +// `llama` target via target_sources() (see llama/server/CMakeLists.txt). The +// matching registration hooks — the arch enum, KV/tensor maps, the model +// factory entry, the rope-type entry, the model class declaration in +// src/models/models.h, and the tokenizer pre-type — are applied to the fetched +// llama.cpp source by llama/models/llama-cpp-laguna.patch. +// +// Keeping the implementation here (rather than as a new file inside the patch) +// means it never conflicts on a llama.cpp version bump; only the small +// registration patch may need regenerating. See llama/models/README.md. +// +// Architecture summary (a dense+MoE decoder): +// * per-head softplus-gated attention output (the "attn_g" projection), +// * Q/K RMSNorm, +// * a sliding-window-attention pattern where full-attention layers use a +// partial-rotary YaRN RoPE and sliding layers use a plain RoPE, +// * a sigmoid-routed MoE (with e_score_correction bias) plus one shared +// expert, with a leading dense block. + +#include "models/models.h" + +void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + // MoE + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); + + // Sliding-window attention. The per-layer attention type is given by + // ".attention.layer_types" (1 == sliding-window, 0 == full attention). + ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); + hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; + + // Per-layer attention type: layer_types[il] == 1 marks a sliding-window + // layer, which maps directly onto swa_layers (1 == SWA, 0 == full). + ml.get_key_or_arr("laguna.attention.layer_types", hparams.swa_layers, hparams.n_layer, false); + + // Dual RoPE. n_rot_full / rope_freq_base_train (full-attention, YaRN) are + // loaded generically; the sliding-window variant uses laguna-specific keys. + // The full-attention YaRN betas reuse llama.cpp's existing yarn_beta_* fields + // (whose 32/1 defaults already match Laguna), so no new hparams are needed. + ml.get_key("laguna.rope.swa.dimension_count", hparams.n_rot_swa, false); + ml.get_key("laguna.rope.swa.freq_base", hparams.rope_freq_base_train_swa, false); + ml.get_key("laguna.rope.scaling.beta_fast", hparams.yarn_beta_fast, false); + ml.get_key("laguna.rope.scaling.beta_slow", hparams.yarn_beta_slow, false); + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_laguna::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_shexp = hparams.n_ff_shexp; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + const int64_t n_head_i = hparams.n_head(i); + const int64_t n_head_kv_i = hparams.n_head_kv(i); + const int64_t n_embd_q = n_embd_head_k * n_head_i; + const int64_t n_embd_kv = n_embd_head_k * n_head_kv_i; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_q}, 0); + layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_kv}, 0); + layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_kv}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); + + // per-head attention output gate (stored on disk as "attn_g") + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE_LAGUNA, "weight", i), {n_embd, n_head_i}, 0); + + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + } + } +} + +std::unique_ptr llama_model_laguna::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + const int64_t n_head_il = hparams.n_head(il); + const int64_t n_head_kv_il = hparams.n_head_kv(il); + const bool is_swa = hparams.is_swa(il); + + // full-attention layers use partial-rotary YaRN; sliding layers use plain RoPE + const int rope_n_dims = hparams.n_rot(il); + const float rope_base = is_swa ? hparams.rope_freq_base_train_swa : hparams.rope_freq_base_train; + const float rope_scale = is_swa ? hparams.rope_freq_scale_train_swa : hparams.rope_freq_scale_train; + const float rope_ext = is_swa ? 0.0f : 1.0f; + const float rope_bfast = hparams.yarn_beta_fast; + const float rope_bslow = hparams.yarn_beta_slow; + + // norm + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // self-attention + { + ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + // per-head output gate, from the same normed input + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur); + cb(gate, "gate", il); + + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head_il, n_tokens); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv_il, n_tokens); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv_il, n_tokens); + + Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); + cb(Qcur, "Qcur_normed", il); + Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); + cb(Kcur, "Kcur_normed", il); + + Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, + rope_n_dims, rope_type, hparams.n_ctx_orig_yarn, rope_base, rope_scale, + rope_ext, hparams.rope_attn_factor, rope_bfast, rope_bslow); + Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, + rope_n_dims, rope_type, hparams.n_ctx_orig_yarn, rope_base, rope_scale, + rope_ext, hparams.rope_attn_factor, rope_bfast, rope_bslow); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + // attention without the output projection: the gate is applied first + cur = build_attn(inp_attn, + nullptr, nullptr, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_pregate", il); + + // per-head gate: softplus over the gate logits, broadcast across head_dim + gate = ggml_softplus(ctx0, gate); + cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_il, n_tokens); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head_il, n_tokens); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_il, n_tokens); + cb(cur, "attn_gated", il); + + cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward: leading dense block(s), MoE + shared expert otherwise + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + model.layers[il].ffn_up, NULL, NULL, + model.layers[il].ffn_gate, NULL, NULL, + model.layers[il].ffn_down, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + model.layers[il].ffn_up_shexp, NULL, NULL, + model.layers[il].ffn_gate_shexp, NULL, NULL, + model.layers[il].ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/llama/models/llama-cpp-laguna.patch b/llama/models/llama-cpp-laguna.patch new file mode 100644 index 000000000..fe8308a92 --- /dev/null +++ b/llama/models/llama-cpp-laguna.patch @@ -0,0 +1,139 @@ +diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp +index e95ba6daa..daff484e3 100644 +--- a/src/llama-arch.cpp ++++ b/src/llama-arch.cpp +@@ -134,6 +134,7 @@ static const std::map LLM_ARCH_NAMES = { + { LLM_ARCH_MAINCODER, "maincoder" }, + { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, + { LLM_ARCH_TALKIE, "talkie" }, ++ { LLM_ARCH_LAGUNA, "laguna" }, + { LLM_ARCH_UNKNOWN, "(unknown)" }, + }; + +@@ -370,6 +371,7 @@ static const std::map LLM_TENSOR_NAMES = { + { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" }, + { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" }, + { LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" }, ++ { LLM_TENSOR_ATTN_GATE_LAGUNA, "blk.%d.attn_g" }, + { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" }, + { LLM_TENSOR_FFN_POST_NORM_1, "blk.%d.post_ffw_norm_1" }, + { LLM_TENSOR_FFN_POST_NORM_2, "blk.%d.post_ffw_norm_2" }, +@@ -585,6 +587,7 @@ static const std::map LLM_TENSOR_INFOS = { + {LLM_TENSOR_ATTN_QKV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_OUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, ++ {LLM_TENSOR_ATTN_GATE_LAGUNA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, +diff --git a/src/llama-arch.h b/src/llama-arch.h +index 7c1dcc4d6..c726452f8 100644 +--- a/src/llama-arch.h ++++ b/src/llama-arch.h +@@ -138,6 +138,7 @@ enum llm_arch { + LLM_ARCH_MAINCODER, + LLM_ARCH_KIMI_LINEAR, + LLM_ARCH_TALKIE, ++ LLM_ARCH_LAGUNA, + LLM_ARCH_UNKNOWN, + }; + +@@ -372,6 +373,7 @@ enum llm_tensor { + LLM_TENSOR_ATTN_ROT_EMBD, + LLM_TENSOR_ATTN_SINKS, + LLM_TENSOR_ATTN_GATE, ++ LLM_TENSOR_ATTN_GATE_LAGUNA, + LLM_TENSOR_FFN_GATE_INP, + LLM_TENSOR_FFN_GATE_INP_SHEXP, + LLM_TENSOR_FFN_NORM, +diff --git a/src/llama-model.cpp b/src/llama-model.cpp +index 0c3e03a61..366d82730 100644 +--- a/src/llama-model.cpp ++++ b/src/llama-model.cpp +@@ -286,6 +286,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params + return new llama_model_kimi_linear(params); + case LLM_ARCH_STEP35: + return new llama_model_step35(params); ++ case LLM_ARCH_LAGUNA: ++ return new llama_model_laguna(params); + default: + throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'"); + } +@@ -2356,6 +2358,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { + case LLM_ARCH_MIMO2: + case LLM_ARCH_STEP35: + case LLM_ARCH_TALKIE: ++ case LLM_ARCH_LAGUNA: + return LLAMA_ROPE_TYPE_NEOX; + + case LLM_ARCH_QWEN2VL: +diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp +index 473becade..898f6e080 100644 +--- a/src/llama-vocab.cpp ++++ b/src/llama-vocab.cpp +@@ -358,6 +358,12 @@ struct llm_tokenizer_bpe : llm_tokenizer { + "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", + }; + break; ++ case LLAMA_VOCAB_PRE_TYPE_LAGUNA: ++ regex_exprs = { ++ "(?:\\r?\\n)+(?!\\r?\\n)", ++ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", ++ }; ++ break; + case LLAMA_VOCAB_PRE_TYPE_GPT2: + case LLAMA_VOCAB_PRE_TYPE_MPT: + case LLAMA_VOCAB_PRE_TYPE_OLMO: +@@ -2050,6 +2056,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { + } else if (tokenizer_pre == "minicpm5") { + pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5; + ignore_merges = true; ++ } else if (tokenizer_pre == "laguna") { ++ pre_type = LLAMA_VOCAB_PRE_TYPE_LAGUNA; + } else if ( + tokenizer_pre == "llama3" || + tokenizer_pre == "llama-v3" || +@@ -2697,6 +2705,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { + || t.first == "" // gemma4 + || t.first == "<|tool_response>" // gemma4 + || t.first == "<｜end▁of▁sentence｜>" // deepseek-ocr ++ || t.first == "" // poolside Laguna (eos_token_ids) + ) { + special_eog_ids.insert(t.second); + if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { +diff --git a/src/llama-vocab.h b/src/llama-vocab.h +index 8ab775942..b69be9154 100644 +--- a/src/llama-vocab.h ++++ b/src/llama-vocab.h +@@ -61,6 +61,7 @@ enum llama_vocab_pre_type { + LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, + LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, + LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, ++ LLAMA_VOCAB_PRE_TYPE_LAGUNA = 53, + }; + + struct LLM_KV; +diff --git a/src/models/models.h b/src/models/models.h +index db228865d..dab40e4f8 100644 +--- a/src/models/models.h ++++ b/src/models/models.h +@@ -1453,6 +1453,19 @@ struct llama_model_dots1 : public llama_model_base { + }; + + ++struct llama_model_laguna : public llama_model_base { ++ llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {} ++ void load_arch_hparams(llama_model_loader & ml) override; ++ void load_arch_tensors(llama_model_loader & ml) override; ++ ++ struct graph : public llm_graph_context { ++ graph(const llama_model & model, const llm_graph_params & params); ++ }; ++ ++ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; ++}; ++ ++ + struct llama_model_arcee : public llama_model_base { + llama_model_arcee(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; diff --git a/llama/models/models.cmake b/llama/models/models.cmake new file mode 100644 index 000000000..788d7c913 --- /dev/null +++ b/llama/models/models.cmake @@ -0,0 +1,28 @@ +# Interim model-architecture support — CMake integration +# +# Mirrors llama/compat/compat.cmake. This directory holds interim, Ollama-owned +# implementations of model architectures not yet available in the pinned +# llama.cpp, plus a small per-architecture registration patch for each: +# +# 1. .cpp - the architecture implementation, linked into the +# fetched llama target (kept in Ollama's tree so it +# never conflicts on a llama.cpp version bump). +# 2. llama-cpp-.patch - the registration hooks: the arch enum + name, +# the model factory and rope-type entries, and the +# model class declaration; plus, only when the model +# needs them, new tensor names, a tokenizer pre-type, +# or hparams fields. Applied to the fetched source. +# +# llama/server/CMakeLists.txt applies every *.patch here (after the compat +# hooks patch) and links every *.cpp here into the fetched llama target. +# +# See llama/models/README.md for how to add a new architecture. + +set(_models_dir ${CMAKE_CURRENT_LIST_DIR}) + +# Directory holding registration patches (*.patch) and architecture sources +# (*.cpp). Exposed so llama/server/CMakeLists.txt can apply the patches and +# link the sources. +set(OLLAMA_LLAMA_CPP_MODELS_DIR + "${_models_dir}" + CACHE INTERNAL "Directory of Ollama llama.cpp architecture sources and patches") diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt index b7f79ddff..aa90bee60 100644 --- a/llama/server/CMakeLists.txt +++ b/llama/server/CMakeLists.txt @@ -129,6 +129,15 @@ endif() # apply the patch by hand while iterating on the compat layer. set(_ollama_compat_patch_cmd "") include(${CMAKE_CURRENT_SOURCE_DIR}/../compat/compat.cmake) +include(${CMAKE_CURRENT_SOURCE_DIR}/../models/models.cmake) + +# Combined patch command: the compat hooks patch plus every new-architecture +# registration patch in llama/models/, applied to the fetched llama.cpp source. +set(OLLAMA_LLAMA_CPP_PATCH_COMMAND + ${CMAKE_COMMAND} + -DPATCH_FILE=${OLLAMA_LLAMA_CPP_COMPAT_PATCH_FILE} + -DPATCH_DIR=${OLLAMA_LLAMA_CPP_MODELS_DIR} + -P ${OLLAMA_LLAMA_CPP_COMPAT_DIR}/apply-patch.cmake) option(OLLAMA_LLAMA_CPP_SKIP_COMPAT_PATCH "Skip llama.cpp compat patch application because the source is already prepared" OFF) @@ -144,7 +153,7 @@ elseif(NOT DEFINED ENV{OLLAMA_LLAMA_CPP_SOURCE}) "Applying Ollama llama.cpp compat patch to source override: " "${_llama_cpp_source_override}") execute_process( - COMMAND ${OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND} + COMMAND ${OLLAMA_LLAMA_CPP_PATCH_COMMAND} WORKING_DIRECTORY "${_llama_cpp_source_override}" RESULT_VARIABLE _ollama_compat_patch_result ) @@ -154,7 +163,7 @@ elseif(NOT DEFINED ENV{OLLAMA_LLAMA_CPP_SOURCE}) "${_llama_cpp_source_override}") endif() else() - set(_ollama_compat_patch_cmd PATCH_COMMAND ${OLLAMA_LLAMA_CPP_COMPAT_PATCH_COMMAND}) + set(_ollama_compat_patch_cmd PATCH_COMMAND ${OLLAMA_LLAMA_CPP_PATCH_COMMAND}) endif() endif() @@ -202,6 +211,21 @@ if(_ollama_link_compat_sources AND DEFINED OLLAMA_LLAMA_CPP_COMPAT_DIR) endif() endif() +# Link the Ollama-owned new-architecture implementations (llama/models/*.cpp) +# into the fetched llama target. Gated on the same condition as the compat +# sources: when iterating against a local llama.cpp via OLLAMA_LLAMA_CPP_SOURCE, +# the developer wires these in by hand. +if(_ollama_link_compat_sources AND DEFINED OLLAMA_LLAMA_CPP_MODELS_DIR) + file(GLOB _ollama_model_sources CONFIGURE_DEPENDS + ${OLLAMA_LLAMA_CPP_MODELS_DIR}/*.cpp) + if(_ollama_model_sources AND TARGET llama) + target_sources(llama PRIVATE ${_ollama_model_sources}) + target_include_directories(llama PRIVATE + ${OLLAMA_LLAMA_CPP_MODELS_DIR} + ${llama_cpp_SOURCE_DIR}/src) + endif() +endif() + # Find GPU toolkits for runtime dependency bundling. # The llama.cpp build finds these internally, but we need the # variables (CUDAToolkit_LIBRARY_DIR, etc.) in our install scope.