mirror of
https://github.com/ollama/ollama.git
synced 2026-09-21 13:55:04 +00:00
8713570d3c
Vision towers are much more sensitive to weight quantization than language layers: measured against the reference encoder on a real image, 4-bit types and scale-only mxfp8 distort the projected image features by 26-34% mean relative error (worst tokens near-orthogonal), which shows up as degraded image recognition — down to complete blindness for the small e-series towers under nvfp4. Affine 8-bit was the only quantized format that matched the bf16 tower. Keep vision tower tensors at source precision instead, matching the audio tower's treatment and every vision component Ollama publishes in GGUF form, including gemma4's own GGUF tags, which ship f16/f32 vision beside 4-bit language weights. Towers are small and run once per image, so neither size nor decode bandwidth argues for quantizing. Existing MLX imports keep their quantized towers until re-imported.
95 lines
3 KiB
Go
95 lines
3 KiB
Go
package create
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
type gemma4ImportTransform struct {
|
|
numLayers int
|
|
numExperts int
|
|
}
|
|
|
|
// gemma4Config is a minimal subset of the Gemma 4 config.json used for quant decisions.
|
|
type gemma4Config struct {
|
|
NumHiddenLayers int `json:"num_hidden_layers"`
|
|
NumExperts int `json:"num_experts"`
|
|
TextConfig struct {
|
|
NumHiddenLayers int `json:"num_hidden_layers"`
|
|
NumExperts int `json:"num_experts"`
|
|
} `json:"text_config"`
|
|
}
|
|
|
|
func newGemma4ImportTransform(rawConfig json.RawMessage) (quantizePolicy, error) {
|
|
var cfg gemma4Config
|
|
if err := json.Unmarshal(rawConfig, &cfg); err != nil {
|
|
return nil, fmt.Errorf("gemma4: parse config.json: %w", err)
|
|
}
|
|
|
|
numLayers := cfg.NumHiddenLayers
|
|
if numLayers == 0 {
|
|
numLayers = cfg.TextConfig.NumHiddenLayers
|
|
}
|
|
numExperts := cfg.NumExperts
|
|
if numExperts == 0 {
|
|
numExperts = cfg.TextConfig.NumExperts
|
|
}
|
|
|
|
return gemma4ImportTransform{numLayers: numLayers, numExperts: numExperts}, nil
|
|
}
|
|
|
|
func (t gemma4ImportTransform) quantizationType(name string, shape []int32, quantize string) string {
|
|
base := normalizeQuantType(quantize)
|
|
switch {
|
|
case isEmbedTokensWeight(name):
|
|
// The embedding doubles as the lm_head projection; an 8-bit type keeps
|
|
// quality close to bf16 (matching GGUF Q6_K) while saving bandwidth.
|
|
// Fall back to the base type when 8-bit does not fit the vocab shape.
|
|
if e := promoteEmbedding(shape, base); e != "" {
|
|
return e
|
|
}
|
|
if isAligned(shape, base) {
|
|
return base
|
|
}
|
|
return ""
|
|
case t.isSensitiveProjection(name) && eightBit(base) != base:
|
|
return sensitiveType(t.promoteSensitive(name), shape, base)
|
|
default:
|
|
// Routing gates, norms, embeddings, etc. are handled by the generic
|
|
// policy; everything else quantizes at the requested type.
|
|
return GetTensorQuantization(name, shape, quantize)
|
|
}
|
|
}
|
|
|
|
// isSensitiveProjection reports the value/key/down projections whose precision
|
|
// most affects quality — attention output (v/k) and the residual stream
|
|
// (down). Audio and vision tensors are excluded and follow the generic
|
|
// policy.
|
|
func (t gemma4ImportTransform) isSensitiveProjection(name string) bool {
|
|
if isVision(name) || isAudioTower(name) {
|
|
return false
|
|
}
|
|
return strings.Contains(name, ".v_proj") ||
|
|
strings.Contains(name, ".k_proj") ||
|
|
strings.Contains(name, "down_proj")
|
|
}
|
|
|
|
// promoteSensitive decides whether a sensitive projection uses 8-bit precision.
|
|
// 8-expert models share very few KV heads, so their k/v projections are always
|
|
// promoted; otherwise v/down projections are promoted at the input and output
|
|
// layers and periodically between (useMoreBits), where residual-stream error
|
|
// accumulates most.
|
|
func (t gemma4ImportTransform) promoteSensitive(name string) bool {
|
|
if t.numLayers == 0 {
|
|
return false
|
|
}
|
|
if t.numExperts == 8 && (strings.Contains(name, ".v_proj") || strings.Contains(name, ".k_proj")) {
|
|
return true
|
|
}
|
|
if strings.Contains(name, ".k_proj") {
|
|
return false // k_proj is promoted only via the 8-expert path
|
|
}
|
|
layer := layerIndex(name)
|
|
return layer >= 0 && useMoreBits(layer, t.numLayers)
|
|
}
|