From 5ec58043600a0ce3f78c0bdde9e6e171df8e1a2c Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Tue, 1 Sep 2026 16:24:15 -0700 Subject: [PATCH] llama.cpp: version bump b10729 (#18160) * llama.cpp: version bump b10729 Regenerate the compat hooks patch for b10729: upstream removed the whole-tensor load_data_for read (last consumer was llama-quantize, which now reads slabs via load_data_range). Keep the existing hook surface (constructor, skip loops, load_all_data, mtmd/clip) unchanged and add maybe_load_text_tensor_range, which materializes a text load op's output once per tensor and serves the new (offset, size) slab reads from that cache. * address comments --- LLAMA_CPP_VERSION | 2 +- llama/compat/001-llama-cpp-hooks.patch | 35 ++++----- llama/compat/README.md | 11 ++- llama/compat/llama-ollama-compat.cpp | 65 +++++++++++++++++ llama/compat/llama-ollama-compat.h | 16 +++++ llama/server/CMakeLists.txt | 98 ++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 19 deletions(-) diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSION index b08661a43..9da8f62c7 100644 --- a/LLAMA_CPP_VERSION +++ b/LLAMA_CPP_VERSION @@ -1 +1 @@ -b10630 +b10729 diff --git a/llama/compat/001-llama-cpp-hooks.patch b/llama/compat/001-llama-cpp-hooks.patch index 3ded2a5e5..661005209 100644 --- a/llama/compat/001-llama-cpp-hooks.patch +++ b/llama/compat/001-llama-cpp-hooks.patch @@ -1,5 +1,5 @@ diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp -index df8313e81..29ccb0f85 100644 +index 7663797ba..4140399a5 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -5,6 +5,7 @@ @@ -10,7 +10,7 @@ index df8313e81..29ccb0f85 100644 #include #include -@@ -561,6 +562,9 @@ llama_model_loader::llama_model_loader( +@@ -574,6 +575,9 @@ llama_model_loader::llama_model_loader( } get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); @@ -20,7 +20,7 @@ index df8313e81..29ccb0f85 100644 llm_kv = LLM_KV(llm_arch_from_string(arch_name)); files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io)); -@@ -571,6 +575,9 @@ llama_model_loader::llama_model_loader( +@@ -584,6 +588,9 @@ llama_model_loader::llama_model_loader( // so we build a unified tensors index for weights. for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { std::string tensor_name = std::string(cur->name); @@ -30,7 +30,7 @@ index df8313e81..29ccb0f85 100644 // make sure there is no duplicated tensor names if (weights_map.find(tensor_name) != weights_map.end()) { throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); -@@ -681,6 +688,9 @@ llama_model_loader::llama_model_loader( +@@ -694,6 +701,9 @@ llama_model_loader::llama_model_loader( // Save tensors data offset info of the main file. for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { std::string tensor_name = std::string(cur->name); @@ -40,24 +40,27 @@ index df8313e81..29ccb0f85 100644 // make sure there is no duplicated tensor names if (weights_map.find(tensor_name) != weights_map.end()) { throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); -@@ -1380,6 +1390,7 @@ void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void * +@@ -1464,6 +1474,10 @@ void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const { + const void * llama_model_loader::load_data_range(const llama_tensor_weight & w, size_t offs, size_t size, void * buf) const { + GGML_ASSERT(offs + size <= ggml_nbytes(w.tensor)); - void llama_model_loader::load_data_for(struct ggml_tensor * cur) const { - const auto & w = require_weight(ggml_get_name(cur)); -+ if (llama_ollama_compat::maybe_load_text_tensor(this, cur, w.offs)) return; ++ if (const void * compat_data = llama_ollama_compat::maybe_load_text_tensor_range(this, w.tensor, offs, size, buf)) { ++ return compat_data; ++ } ++ + const void * data = buf; if (use_mmap) { - const auto & mapping = mappings.at(w.idx); -@@ -1530,6 +1541,7 @@ bool llama_model_loader::load_all_data( +@@ -1612,6 +1626,7 @@ bool llama_model_loader::load_all_data( } size_t n_size = ggml_nbytes(cur); + if (llama_ollama_compat::maybe_load_text_tensor(this, cur, weight->offs)) continue; - if (use_mmap) { - const auto & mapping = mappings.at(weight->idx); + const bool from_mapping = use_mmap || lazy.has(cur); + diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp -index c1870813f..a1a923082 100644 +index 90de19575..5a8e4c675 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -10,6 +10,8 @@ @@ -69,7 +72,7 @@ index c1870813f..a1a923082 100644 #include #include #include -@@ -1093,6 +1095,11 @@ struct clip_model_loader { +@@ -1188,6 +1190,11 @@ struct clip_model_loader { ctx_meta.reset(meta); @@ -81,7 +84,7 @@ index c1870813f..a1a923082 100644 const int n_tensors = gguf_get_n_tensors(ctx_gguf.get()); // print gguf info -@@ -3085,6 +3092,7 @@ struct clip_model_loader { +@@ -3569,6 +3576,7 @@ struct clip_model_loader { auto it_off = tensor_offset.find(t->name); GGML_ASSERT(it_off != tensor_offset.end() && "no offset for tensor"); const size_t offset = it_off->second; @@ -89,7 +92,7 @@ index c1870813f..a1a923082 100644 fin.seekg(offset, std::ios::beg); if (!fin) { throw std::runtime_error(string_format("%s: failed to seek for tensor %s\n", __func__, t->name)); -@@ -4964,6 +4972,15 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 +@@ -5797,6 +5805,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } int clip_n_mmproj_embd(const struct clip_ctx * ctx) { diff --git a/llama/compat/README.md b/llama/compat/README.md index eafae60cc..caf3fbcfd 100644 --- a/llama/compat/README.md +++ b/llama/compat/README.md @@ -53,8 +53,15 @@ The layer runs at a small set of loader hook points: vision, audio, MTP, or other tensors that the text loader should not claim. 3. Main model tensor reads: `maybe_load_text_tensor` applies registered text-side load operations, such as FFN concat or dtype promotion, before - the normal llama.cpp file read. This is wired into both full model loading - and single-tensor reads used by tools such as `llama-quantize`. + the normal llama.cpp file read. This is wired into full model loading + (`load_all_data`) and single-tensor reads used by tools such as + `llama-quantize`. Since llama.cpp b10729 replaced the whole-tensor + `load_data_for` read with slabs via `load_data_range`, the quantize-style + slab path goes through `maybe_load_text_tensor_range`, which materializes + the op's full output for one active tensor at a time (single-slot cache — + evicted when the next tensor's first range arrives) and serves each + requested (offset, size) range from it, so quantize memory stays at one + op tensor, matching the whole-tensor `load_data_for` read it replaced. 4. `mtmd/clip` constructor: `translate_clip_metadata` rewrites a clip-facing view of monolithic GGUFs into the mmproj form expected by llama.cpp. 5. `mtmd/clip` tensor load loop: `maybe_load_tensor` applies clip-side load diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp index ce5abd59e..0888fef0e 100644 --- a/llama/compat/llama-ollama-compat.cpp +++ b/llama/compat/llama-ollama-compat.cpp @@ -3438,6 +3438,71 @@ bool maybe_load_text_tensor(const llama_model_loader * ml, return load_tensor_with_op(cur, path.c_str(), buft, op); } +// Slab-read cache slot (maybe_load_text_tensor_range). Holds AT MOST ONE +// materialized tensor per loader: quantize reads a tensor's slabs +// contiguously, so the previous entry is evicted when the next tensor's +// first range arrives. This keeps peak memory at one op tensor at a time — +// the same profile as the whole-tensor read this hook replaced. Growing a +// per-tensor map instead would accumulate every layer's output for per-layer +// ops (gemma4 MoE gate/up, qwen3.5 norm-shift) — most of the model in RAM by +// the end of a quantize run. +std::mutex g_text_range_mutex; +std::unordered_map>> + g_text_range_cache; + +const void * maybe_load_text_tensor_range(const llama_model_loader * ml, + ggml_tensor * cur, + size_t offs, + size_t size, + void * buf) { + if (compat_disabled()) return nullptr; + std::string path; + { + std::lock_guard lk(g_loader_path_mutex); + auto it = g_loader_paths.find(ml); + if (it == g_loader_paths.end() || it->second.empty()) return nullptr; + path = it->second; + } + + std::lock_guard lk(g_text_range_mutex); + auto & slot = g_text_range_cache[ml]; + auto cached = slot.first == ggml_get_name(cur) ? &slot.second : nullptr; + if (!cached) { + LoadOp op; + if (!take_load_op(ggml_get_name(cur), op)) return nullptr; + + const auto start = std::chrono::steady_clock::now(); + std::vector full(ggml_nbytes(cur)); + if (!op.apply(path.c_str(), full.data(), full.size())) { + OLLAMA_COMPAT_LOG_ERROR("%s: %s failed for %s after %.3f ms\n", + __func__, op.description, ggml_get_name(cur), elapsed_ms(start)); + return nullptr; + } + + const size_t dst_size = full.size(); + slot = {std::string(ggml_get_name(cur)), std::move(full)}; + cached = &slot.second; + const double ms = elapsed_ms(start); + const TransformTiming total = record_transform_timing(dst_size, ms); + OLLAMA_COMPAT_LOG_INFO("compat tensor transform: op=%s tensor=%s bytes=%zu duration_ms=%.3f total_ops=%llu total_bytes=%zu total_ms=%.3f\n", + op.description, ggml_get_name(cur), dst_size, ms, + (unsigned long long) total.count, total.bytes, total.ms); + } + + if (offs + size > cached->size()) { + OLLAMA_COMPAT_LOG_ERROR("%s: range %zu+%zu out of bounds for %s (%zu bytes)\n", + __func__, offs, size, ggml_get_name(cur), cached->size()); + return nullptr; + } + + const void * out = buf ? buf : cached->data() + offs; + if (buf) { + std::memcpy(buf, cached->data() + offs, size); + } + return out; +} + int maybe_clip_mmproj_embd(const char * projector_type, int projection_dim) { if (compat_disabled() || projection_dim <= 0) return 0; if (!clip_mmproj_embd_uses_projection_dim(projector_type)) return 0; diff --git a/llama/compat/llama-ollama-compat.h b/llama/compat/llama-ollama-compat.h index c97326df4..ba3f06096 100644 --- a/llama/compat/llama-ollama-compat.h +++ b/llama/compat/llama-ollama-compat.h @@ -79,6 +79,22 @@ bool maybe_load_text_tensor(const llama_model_loader * ml, ggml_tensor * cur, size_t file_offset); +// Slab-serving variant for loaders that read tensor data in (offset, size) +// byte ranges (llama_model_loader::load_data_range, used by llama-quantize +// and other single-tensor read tools). Same registry and file-path lookup +// as maybe_load_text_tensor, but serves one range of the op's destination +// bytes per call: the first call for a tensor materializes the op's full +// output, and later ranges are served from that cache. To keep quantize +// memory at a single op tensor (the same profile as the whole-tensor read +// this hook replaced), the cache holds one active tensor per loader. +// Returns the requested range (copied into buf when buf is non-null) or +// nullptr when no load op exists for this tensor. +const void * maybe_load_text_tensor_range(const llama_model_loader * ml, + ggml_tensor * cur, + size_t offs, + size_t size, + void * buf); + // Called from clip_n_mmproj_embd() before the upstream switch. Returns a // positive embedding size only for Ollama compatibility cases whose projector // metadata already follows upstream naming, but whose legacy projector type diff --git a/llama/server/CMakeLists.txt b/llama/server/CMakeLists.txt index 7d88fcd7d..b994b14bd 100644 --- a/llama/server/CMakeLists.txt +++ b/llama/server/CMakeLists.txt @@ -187,6 +187,85 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(llama_cpp) +if(NOT COMMAND license_add_file) + message(FATAL_ERROR "llama.cpp license registry (cmake/license.cmake) not found; check the llama.cpp update") +endif() +set(_llama_cpp_root "${llama_cpp_SOURCE_DIR}") + +file(GLOB_RECURSE _ollama_vendor_license_files + "${_llama_cpp_root}/vendor/*/LICENSE*") +file(GLOB _llamacpp_repo_license_files + "${_llama_cpp_root}/licenses/LICENSE*") +list(APPEND _ollama_vendor_license_files ${_llamacpp_repo_license_files}) +list(APPEND _ollama_vendor_license_files "${_llama_cpp_root}/LICENSE") +foreach(_lic IN LISTS _ollama_vendor_license_files) + cmake_path(RELATIVE_PATH _lic BASE_DIRECTORY "${_llama_cpp_root}") + # upstream registers these itself; skip to avoid double entries + if(_lic STREQUAL "LICENSE" OR _lic STREQUAL "vendor/cpp-httplib/LICENSE") + continue() + endif() + set(_name "${_lic}") + if(_name MATCHES "^vendor/") + # registry name = component path without the vendor/ prefix + string(REGEX REPLACE "^vendor/" "" _name "${_name}") + string(REGEX REPLACE "/LICENSE[^/]*$" "" _name "${_name}") + endif() + string(REPLACE "/" "-" _name "${_name}") + license_add_file("${_name}" "${_llama_cpp_root}/${_lic}") +endforeach() +unset(_ollama_vendor_license_files) + +function(ollama_extract_vendor_license name src start_marker end_marker) + if(NOT EXISTS "${src}") + message(FATAL_ERROR + "llama.cpp vendored source missing: ${src}; check the llama.cpp update " + "and the embedded-license entry in llama/server/CMakeLists.txt") + endif() + file(READ "${src}" _txt) + string(FIND "${_txt}" "${start_marker}" _start) + if(_start EQUAL -1) + message(FATAL_ERROR + "llama.cpp embedded license anchor '${start_marker}' not found in " + "${src}; update llama/server/CMakeLists.txt") + endif() + string(FIND "${_txt}" "${end_marker}" _end) + if(_end EQUAL -1) + message(FATAL_ERROR + "llama.cpp embedded license end anchor '${end_marker}' not found in " + "${src}; update llama/server/CMakeLists.txt") + endif() + string(LENGTH "${end_marker}" _end_len) + math(EXPR _len "${_end} - ${_start} + ${_end_len}") + string(SUBSTRING "${_txt}" "${_start}" "${_len}" _lic) + set(_dst "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/${name}.txt") + file(WRITE "${_dst}" "${_lic}") + license_add_file("${name}" "${_dst}") +endfunction() + +ollama_extract_vendor_license( + "llamafile-sgemm" + "${_llama_cpp_root}/ggml/src/ggml-cpu/llamafile/sgemm.cpp" + "Copyright 2024 Mozilla Foundation" + "// SOFTWARE.") + +get_property(_reg_text GLOBAL PROPERTY LICENSE_TEXT) +string(REPLACE "R\"=L=(" "" _reg_text "${_reg_text}") +string(REPLACE ")=L=\"," "\n" _reg_text "${_reg_text}") +set(_reg_ok TRUE) +foreach(_anchor IN ITEMS "llama.cpp" "cpp-httplib" "llamafile-sgemm" "jsonhpp") + string(FIND "${_reg_text}" "${_anchor}" _pos) + if(_pos EQUAL -1) + set(_reg_ok FALSE) + endif() +endforeach() +string(LENGTH "${_reg_text}" _reg_len) +if(NOT _reg_ok OR _reg_len LESS 256) + message(FATAL_ERROR "llama.cpp vendored license aggregate came out empty or incomplete; the registry mechanism changed upstream") +endif() +file(WRITE "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE" "${_reg_text}") +unset(_reg_text) +unset(_llama_cpp_root) + # Link the Ollama-compat source files into the fetched llama target. # Kept separate from the hook patch so our .cpp/.h stay # on-disk in llama/compat/ rather than being copied into _deps/. @@ -539,6 +618,25 @@ else() RUNTIME DESTINATION ${_base_dest} COMPONENT llama-server OPTIONAL LIBRARY DESTINATION ${_base_dest} COMPONENT llama-server OPTIONAL) + get_target_property(_llama_cpp_license_source llama SOURCE_DIR) + if(NOT _llama_cpp_license_source OR _llama_cpp_license_source MATCHES "-NOTFOUND$") + message(FATAL_ERROR "llama.cpp source directory not found for license install") + endif() + get_filename_component(_llama_cpp_license_root "${_llama_cpp_license_source}/.." ABSOLUTE) + install(FILES + "${_llama_cpp_license_root}/LICENSE" + DESTINATION ${OLLAMA_LIB_DIR} + RENAME LLAMA_CPP_LICENSE + COMPONENT llama-server) + install(FILES + "${_llama_cpp_license_root}/vendor/cpp-httplib/LICENSE" + DESTINATION ${OLLAMA_LIB_DIR} + RENAME CPP_HTTPLIB_LICENSE + COMPONENT llama-server) + install(FILES + "${CMAKE_BINARY_DIR}/ollama-vendor-licenses/LLAMA_CPP_VENDORS_LICENSE" + DESTINATION ${OLLAMA_LIB_DIR} + COMPONENT llama-server) # Bundle Windows CRT DLLs alongside the executables so zip installs # do not depend on host-global redistributables. ollama_install_windows_runtime_dlls("${_base_dest}")