mirror of
https://github.com/ollama/ollama.git
synced 2026-11-22 00:01:45 +00:00
* mlxrunner: add Inkling 2 Mini * mlx: fuse Inkling 2 Mini residual mixing on Metal Fuse RMS normalization and query scoring in one kernel. Apply the softmax weights in a second kernel to avoid storing per-query products. * create: expose Inkling 2 Mini context length in model metadata Copy model_max_length to max_position_embeddings during import so ollama show reports the checkpoint's context length. Reject configs that specify conflicting limits. * mlxrunner: reduce Metal dispatches in Inkling 2 Mini decode Reuse the prefill mask shader to compute relative position bias in one dispatch for single-query attention. On M3 Ultra with Metal 3, compute the loop controller's gain and forget projections in one matrix multiply.
205 lines
6.4 KiB
Go
205 lines
6.4 KiB
Go
package create
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/ollama/ollama/mlx/quant"
|
|
)
|
|
|
|
// SourceKind is the overarching dtype for a given safetensors model
|
|
type SourceKind int
|
|
|
|
const (
|
|
SourceFloat SourceKind = iota // bf16/fp16/fp32 — quantizable on request
|
|
SourceBlockFP8 // HF block-FP8 — auto-converted to mxfp8
|
|
SourcePrequantized // already quantized — copied through
|
|
)
|
|
|
|
func (k SourceKind) String() string {
|
|
switch k {
|
|
case SourceFloat:
|
|
return "float"
|
|
case SourceBlockFP8:
|
|
return "block-fp8"
|
|
case SourcePrequantized:
|
|
return "prequantized"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Classification is the decision about a source model: its kind and the
|
|
// effective quantization of the imported weights. Quantize may describe a
|
|
// requested conversion or a quantization already present in the source.
|
|
type Classification struct {
|
|
Kind SourceKind
|
|
Quantize string
|
|
}
|
|
|
|
// Classify decides a source model's kind and resolves the effective
|
|
// quantization from the user's requested type, rejecting requests that are not
|
|
// allowed for the kind.
|
|
func Classify(inv Inventory, requested string) (Classification, error) {
|
|
requested, err := normalizeRequested(requested)
|
|
if err != nil {
|
|
return Classification{}, err
|
|
}
|
|
|
|
if name, ok := firstUnsupportedFP8(inv); ok {
|
|
return Classification{}, fmt.Errorf("unsupported fp8 source: tensor %s is F8_E5M2; only F8_E4M3 block-FP8 sources are supported", name)
|
|
}
|
|
|
|
kind := detectKind(inv)
|
|
if inv.Config.Architecture() == inkling2Architecture {
|
|
kind, err = classifyInkling2Banks(inv, kind)
|
|
if err != nil {
|
|
return Classification{}, err
|
|
}
|
|
if kind == SourceFloat && requested != "" {
|
|
return Classification{}, fmt.Errorf("Inkling 2 requires importing the source weights without --quantize (requested %q)", requested)
|
|
}
|
|
}
|
|
switch kind {
|
|
case SourceFloat:
|
|
if requested != "" && inv.Config.Architecture() == "Inkling2ForConditionalGenerationMTP" {
|
|
return Classification{}, fmt.Errorf("Inkling 2 MTP requires importing the draft weights without --draft-quantize (requested %q)", requested)
|
|
}
|
|
return Classification{Kind: SourceFloat, Quantize: requested}, nil
|
|
|
|
case SourcePrequantized:
|
|
if requested != "" {
|
|
return Classification{}, fmt.Errorf("cannot requantize an already-quantized source model (requested %q): only bf16/fp16/fp32 sources can be quantized", requested)
|
|
}
|
|
return Classification{
|
|
Kind: SourcePrequantized,
|
|
Quantize: detectPrequantizedQuantization(inv),
|
|
}, nil
|
|
|
|
case SourceBlockFP8:
|
|
rows, cols, ok := inv.Config.HFFP8WeightBlockSize()
|
|
if !ok {
|
|
return Classification{}, fmt.Errorf("fp8 source model is missing weight_block_size metadata")
|
|
}
|
|
if rows != 128 || cols != 128 {
|
|
return Classification{}, fmt.Errorf("unsupported fp8 source block size %dx%d (only 128x128 is supported)", rows, cols)
|
|
}
|
|
if requested != "" {
|
|
return Classification{}, fmt.Errorf("cannot quantize an fp8 source model (requested %q): fp8 sources are converted to mxfp8 automatically; only bf16/fp16/fp32 sources can be quantized", requested)
|
|
}
|
|
return Classification{Kind: SourceBlockFP8, Quantize: "mxfp8"}, nil
|
|
}
|
|
|
|
return Classification{}, fmt.Errorf("could not classify source model in %s", inv.Dir)
|
|
}
|
|
|
|
// detectPrequantizedQuantization reports a single quantization shared by the
|
|
// source's recognized prequantized weights. A mixed or unrecognized source has
|
|
// no single file type to record in the model manifest.
|
|
func detectPrequantizedQuantization(inv Inventory) string {
|
|
var detected string
|
|
for _, name := range sortedTensorNames(inv) {
|
|
spec, _, ok := matchPrequant(name, inv)
|
|
if !ok {
|
|
continue
|
|
}
|
|
q := quant.Canonical(spec.Metadata["quant_type"])
|
|
if q == "" {
|
|
continue
|
|
}
|
|
if detected != "" && detected != q {
|
|
return ""
|
|
}
|
|
detected = q
|
|
}
|
|
return detected
|
|
}
|
|
|
|
// normalizeRequested validates the user's quantize value and returns its
|
|
// canonical form ("" for no quantization).
|
|
func normalizeRequested(requested string) (string, error) {
|
|
if strings.TrimSpace(requested) == "" {
|
|
return "", nil
|
|
}
|
|
c := quant.Canonical(requested)
|
|
if c == "" {
|
|
return "", fmt.Errorf("unsupported quantize type %q: supported types are int4, int8, nvfp4, mxfp4, mxfp8", requested)
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// detectKind sorts a source into Float, BlockFP8, or Prequantized using only
|
|
// the inventory's tensor names, dtypes, and config. Prequantized is detected
|
|
// from the tensors themselves, so a model whose quantization config sidecar is
|
|
// missing (e.g. a ModelOpt checkpoint without hf_quant_config.json) is still
|
|
// recognized as already-quantized and not mistaken for a float model.
|
|
func detectKind(inv Inventory) SourceKind {
|
|
var hasMLXScales, hasPacked, hasPrequantScale, hasFP8Weight bool
|
|
for name, t := range inv.Tensors {
|
|
switch {
|
|
case strings.HasSuffix(name, ".scales"):
|
|
hasMLXScales = true
|
|
case strings.HasSuffix(name, ".weight_packed"):
|
|
hasPacked = true
|
|
case strings.HasSuffix(name, ".weight_scale"):
|
|
// ModelOpt NVFP4 uses a packed weight with an FP8 scale; its
|
|
// MXFP8 uses an FP8 weight with a U8 scale. HF block-FP8 also
|
|
// has a scale companion but is decoded through a separate path.
|
|
if bt, ok := inv.Tensors[strings.TrimSuffix(name, "_scale")]; ok &&
|
|
(isPackedDtype(bt.Dtype) || isE4M3Dtype(bt.Dtype) && strings.EqualFold(t.Dtype, "U8")) {
|
|
hasPrequantScale = true
|
|
}
|
|
}
|
|
if strings.HasSuffix(name, ".weight") && isE4M3Dtype(t.Dtype) {
|
|
hasFP8Weight = true
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case hasMLXScales || hasPacked || hasPrequantScale:
|
|
return SourcePrequantized
|
|
case hasFP8Weight:
|
|
return SourceBlockFP8
|
|
default:
|
|
return SourceFloat
|
|
}
|
|
}
|
|
|
|
// firstUnsupportedFP8 returns the name of the first F8_E5M2 weight in the
|
|
// source, if any. We decode only E4M3, so an E5M2 source must be rejected
|
|
// explicitly rather than silently mishandled.
|
|
func firstUnsupportedFP8(inv Inventory) (string, bool) {
|
|
for name, t := range inv.Tensors {
|
|
if strings.HasSuffix(name, ".weight") && isE5M2Dtype(t.Dtype) {
|
|
return name, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func isPackedDtype(dtype string) bool {
|
|
switch strings.ToUpper(dtype) {
|
|
case "U8", "U32": // current .weight_scale producers ship U8; U32 covers a future word-packed source
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func isE4M3Dtype(dtype string) bool {
|
|
switch strings.ToUpper(dtype) {
|
|
case "F8_E4M3", "F8_E4M3FN":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func isE5M2Dtype(dtype string) bool {
|
|
switch strings.ToUpper(dtype) {
|
|
case "F8_E5M2", "F8_E5M2FNUZ":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|