diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7d3e6b1ac..7062f570e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -24,7 +24,7 @@ jobs: run: | echo GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${GITHUB_REF_NAME#v}\" \"-X=github.com/ollama/ollama/server.mode=release\"'" | tee -a $GITHUB_OUTPUT echo VERSION="${GITHUB_REF_NAME#v}" | tee -a $GITHUB_OUTPUT - echo vendorsha=$(cat LLAMA_CPP_VERSION) | tee -a $GITHUB_OUTPUT + echo vendorsha=$(cat LLAMA_CPP_VERSION)-$(cat MLX_VERSION)-$(cat MLX_C_VERSION) | tee -a $GITHUB_OUTPUT darwin-build: runs-on: macos-26-xlarge diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7ab092767..38d129037 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,7 +38,21 @@ jobs: | xargs python3 -c "import sys; from pathlib import Path; print(any(Path(x).match(glob) for x in sys.argv[1:] for glob in '$*'.split(' ')))" } - echo changed=$(changed 'llama/server/**/*' 'llama/compat/**/*' 'LLAMA_CPP_VERSION' 'llama/llama.cpp/**/*' 'ml/backend/ggml/ggml/**/*' '.github/**/*') | tee -a $GITHUB_OUTPUT + echo changed=$(changed \ + 'CMakeLists.txt' \ + 'CMakePresets.json' \ + 'cmake/**' \ + 'cmake/**/*' \ + 'llama/server/**/*' \ + 'llama/compat/**/*' \ + 'LLAMA_CPP_VERSION' \ + 'MLX_VERSION' \ + 'MLX_C_VERSION' \ + 'llama/llama.cpp/**/*' \ + 'ml/backend/ggml/ggml/**/*' \ + 'x/imagegen/mlx/**' \ + 'x/imagegen/mlx/**/*' \ + '.github/**/*') | tee -a $GITHUB_OUTPUT echo app_changed=$(changed 'app/**' 'app/**/*') | tee -a $GITHUB_OUTPUT echo enginehash=$(cat LLAMA_CPP_VERSION)-$(cat MLX_VERSION)-$(cat MLX_C_VERSION) | tee -a $GITHUB_OUTPUT @@ -68,32 +82,38 @@ jobs: matrix: include: - preset: CPU - server_preset: cpu - server_build_dir: build/llama-server-cpu + superbuild_target: ollama-local + superbuild_dir: build/local-superbuild + superbuild_args: '-DMLX_ENGINE=OFF' + expected_payload: lib/ollama/llama-server + install-go: true - preset: CUDA container: nvidia/cuda:13.0.0-devel-ubuntu22.04 - flags: '-DCMAKE_CUDA_ARCHITECTURES=87' - server_preset: cuda-v13 - server_build_dir: build/llama-server-cuda-v13 + superbuild_target: ollama-llama-server-cuda-v13 + superbuild_dir: build/local-superbuild-cuda-v13 + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=cuda-v13 -DCMAKE_CUDA_ARCHITECTURES=87' + expected_payload: lib/ollama/cuda_v13/libggml-cuda.so - preset: ROCm container: rocm/dev-ubuntu-22.04:7.2.1 extra-packages: rocm-libs - flags: '-DAMDGPU_TARGETS=gfx1010 -DCMAKE_PREFIX_PATH=/opt/rocm' - server_preset: rocm - server_build_dir: build/llama-server-rocm + superbuild_target: ollama-llama-server-rocm + superbuild_dir: build/local-superbuild-rocm + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=rocm -DAMDGPU_TARGETS=gfx1010 -DCMAKE_PREFIX_PATH=/opt/rocm' + expected_payload: lib/ollama/rocm/libggml-hip.so - preset: Vulkan container: ubuntu:22.04 extra-packages: > mesa-vulkan-drivers vulkan-tools libvulkan1 libvulkan-dev vulkan-sdk cmake ccache g++ make - server_preset: vulkan - server_build_dir: build/llama-server-vulkan + superbuild_target: ollama-llama-server-vulkan + superbuild_dir: build/local-superbuild-vulkan + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=vulkan' + expected_payload: lib/ollama/vulkan/libggml-vulkan.so - preset: 'MLX CUDA 13' container: nvidia/cuda:13.0.0-devel-ubuntu22.04 extra-packages: libcudnn9-dev-cuda-13 libopenblas-dev liblapack-dev liblapacke-dev git curl flags: '-DCMAKE_CUDA_ARCHITECTURES=87 -DMLX_CUDA_ARCHITECTURES=80-virtual -DBLAS_INCLUDE_DIRS=/usr/include/x86_64-linux-gnu -DLAPACK_INCLUDE_DIRS=/usr/include/x86_64-linux-gnu' - install-go: true mlx_preset: 'MLX CUDA 13' runs-on: linux container: ${{ matrix.container }} @@ -122,18 +142,30 @@ jobs: - if: matrix.install-go name: Install Go run: | + [ -n "${{ matrix.container }}" ] || sudo=sudo GO_VERSION=$(awk '/^go / { print $2 }' go.mod) - curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" | tar xz -C /usr/local + curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" | $sudo tar xz -C /usr/local echo "/usr/local/go/bin" >> $GITHUB_PATH - uses: actions/cache@v4 with: path: /github/home/.cache/ccache key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}-${{ needs.changes.outputs.enginehash }} - - name: Build llama-server - if: matrix.server_preset + - name: Build native superbuild + if: matrix.superbuild_target run: | - cmake -S llama/server --preset "${{ matrix.server_preset }}" ${{ matrix.flags }} - cmake --build "${{ matrix.server_build_dir }}" -- -l $(nproc) + cmake -S . -B "${{ matrix.superbuild_dir }}" ${{ matrix.superbuild_args }} + CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) cmake --build "${{ matrix.superbuild_dir }}" --target "${{ matrix.superbuild_target }}" -- -l $(nproc) + test -e "${{ matrix.superbuild_dir }}/${{ matrix.expected_payload }}" + - name: Verify local superbuild install + if: matrix.superbuild_target == 'ollama-local' + run: | + ./ollama --version + "${{ matrix.superbuild_dir }}/lib/ollama/llama-server" --version + test -x "${{ matrix.superbuild_dir }}/lib/ollama/llama-quantize" + cmake --install "${{ matrix.superbuild_dir }}" --component ollama-local --prefix "$RUNNER_TEMP/ollama-local" + "$RUNNER_TEMP/ollama-local/bin/ollama" --version + "$RUNNER_TEMP/ollama-local/lib/ollama/llama-server" --version + test -x "$RUNNER_TEMP/ollama-local/lib/ollama/llama-quantize" - name: Build MLX if: matrix.mlx_preset run: | @@ -147,13 +179,16 @@ jobs: matrix: include: - preset: CPU - server_preset: cpu - server_build_dir: build\llama-server-cpu + superbuild_target: ollama-local + superbuild_dir: build\local-superbuild + superbuild_args: '-DMLX_ENGINE=OFF' + expected_payload: lib\ollama\llama-server.exe - preset: CUDA install: https://developer.download.nvidia.com/compute/cuda/13.0.0/local_installers/cuda_13.0.0_windows.exe - flags: '-DCMAKE_CUDA_ARCHITECTURES=80' - server_preset: cuda-v13-windows - server_build_dir: build\llama-server-cuda-v13 + superbuild_target: ollama-llama-server-cuda-v13-windows + superbuild_dir: build\local-superbuild-cuda-v13 + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=cuda-v13-windows -DCMAKE_CUDA_ARCHITECTURES=80' + expected_payload: lib\ollama\cuda_v13\ggml-cuda.dll cuda-components: - '"cudart"' - '"nvcc"' @@ -166,13 +201,16 @@ jobs: - preset: ROCm install: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-26.Q1-Win11-For-HIP.exe rocm-version: '7.1' - flags: '-DAMDGPU_TARGETS=gfx1010' - server_preset: rocm-windows - server_build_dir: build\llama-server-rocm + superbuild_target: ollama-llama-server-rocm-windows + superbuild_dir: build\local-superbuild-rocm + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=rocm-windows -DAMDGPU_TARGETS=gfx1010' + expected_payload: lib\ollama\rocm\ggml-hip.dll - preset: Vulkan install: https://sdk.lunarg.com/sdk/download/1.4.321.1/windows/vulkansdk-windows-X64-1.4.321.1.exe - server_preset: vulkan - server_build_dir: build\llama-server-vulkan + superbuild_target: ollama-llama-server-vulkan + superbuild_dir: build\local-superbuild-vulkan + superbuild_args: '-DOLLAMA_LLAMA_SERVER_BACKENDS=vulkan' + expected_payload: lib\ollama\vulkan\ggml-vulkan.dll - preset: 'MLX CUDA 13' install: https://developer.download.nvidia.com/compute/cuda/13.0.0/local_installers/cuda_13.0.0_windows.exe cudnn-install: https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-9.18.1.3_cuda13-archive.zip @@ -275,19 +313,44 @@ jobs: C:\Program Files\NVIDIA\CUDNN key: ${{ matrix.install }}-${{ matrix.cudnn-install }} - uses: actions/checkout@v4 + - if: matrix.superbuild_target == 'ollama-local' + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' - uses: actions/cache@v4 with: path: ${{ github.workspace }}\.ccache key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.preset }}-${{ needs.changes.outputs.enginehash }} - - name: Build llama-server - if: matrix.server_preset + - name: Build native superbuild + if: matrix.superbuild_target run: | + $ErrorActionPreference = "Stop" Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll' Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -no_logo' - cmake -S llama\server --preset "${{ matrix.server_preset }}" ${{ matrix.flags }} - cmake --build "${{ matrix.server_build_dir }}" -- -l $([Environment]::ProcessorCount) + cmake -S . -B "${{ matrix.superbuild_dir }}" ${{ matrix.superbuild_args }} + $env:CMAKE_BUILD_PARALLEL_LEVEL = [Environment]::ProcessorCount + cmake --build "${{ matrix.superbuild_dir }}" --target "${{ matrix.superbuild_target }}" -- -l $([Environment]::ProcessorCount) + if (!(Test-Path "${{ matrix.superbuild_dir }}\${{ matrix.expected_payload }}")) { + throw "missing ${{ matrix.expected_payload }}" + } env: CMAKE_GENERATOR: Ninja + - name: Verify local superbuild install + if: matrix.superbuild_target == 'ollama-local' + run: | + $ErrorActionPreference = "Stop" + & ".\ollama.exe" --version + & "${{ matrix.superbuild_dir }}\lib\ollama\llama-server.exe" --version + if (!(Test-Path "${{ matrix.superbuild_dir }}\lib\ollama\llama-quantize.exe")) { + throw "missing llama-quantize.exe" + } + $installPrefix = Join-Path $env:RUNNER_TEMP "ollama-local" + cmake --install "${{ matrix.superbuild_dir }}" --component ollama-local --prefix "$installPrefix" + & "$installPrefix\bin\ollama.exe" --version + & "$installPrefix\lib\ollama\llama-server.exe" --version + if (!(Test-Path "$installPrefix\lib\ollama\llama-quantize.exe")) { + throw "missing installed llama-quantize.exe" + } - name: Build MLX if: matrix.mlx_preset run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 9014ebcfc..4ba22e526 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,11 +32,12 @@ endif() set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS ON) # Recent versions of MLX Requires gnu++17 extensions to compile properly +set(CMAKE_CXX_EXTENSIONS ON) # Recent versions of MLX require gnu++17 extensions to compile properly # GGML backend for inference is provided by llama-server (built separately via -# llama/server/CMakeLists.txt using FetchContent from upstream llama.cpp). -# This root CMakeLists.txt only builds MLX (when enabled). +# llama/server/CMakeLists.txt using FetchContent from the pinned llama.cpp source). +# The root CMake project is the orchestration entrypoint; backend-specific +# build rules live in subprojects under cmake/. if(APPLE) set(CMAKE_BUILD_RPATH "@loader_path") @@ -45,7 +46,7 @@ if(APPLE) endif() set(OLLAMA_BUILD_DIR ${CMAKE_BINARY_DIR}/lib/ollama) -set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/lib/ollama/${OLLAMA_RUNNER_DIR}) +set(OLLAMA_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}/ollama/${OLLAMA_RUNNER_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${OLLAMA_BUILD_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${OLLAMA_BUILD_DIR}) @@ -55,188 +56,9 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${OLLAMA_BUILD_DIR}) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${OLLAMA_BUILD_DIR}) option(MLX_ENGINE "Enable MLX backend" OFF) + +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/local.cmake) + if(MLX_ENGINE) - if(MLX_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES) - check_language(CUDA) - endif() - - message(STATUS "Setting up MLX (this takes a while...)") - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/x/imagegen/mlx) - - # Find CUDA toolkit if MLX is built with CUDA support - find_package(CUDAToolkit) - - # Build list of directories for runtime dependency resolution - set(MLX_RUNTIME_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR}) - # Add cuDNN bin paths for DLLs (Windows MLX CUDA builds) - # CUDNN_ROOT_DIR is the standard CMake variable for cuDNN location - if(DEFINED ENV{CUDNN_ROOT_DIR}) - # cuDNN 9.x has versioned subdirectories under bin/ (e.g., bin/13.0/) - file(GLOB CUDNN_BIN_SUBDIRS "$ENV{CUDNN_ROOT_DIR}/bin/*") - list(APPEND MLX_RUNTIME_DIRS ${CUDNN_BIN_SUBDIRS}) - endif() - # Add build output directory and MLX dependency build directories - list(APPEND MLX_RUNTIME_DIRS ${OLLAMA_BUILD_DIR}) - # OpenBLAS DLL location (pre-built zip extracts into openblas-src/bin/) - list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin) - # NCCL: on Linux, if real NCCL is found, cmake bundles libnccl.so via the - # regex below. If NCCL is not found, MLX links a static stub (OBJECT lib) - # so there is no runtime dependency. This path covers the stub build dir - # for windows so we include the DLL in our dependencies. - list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release) - - # Base regexes for runtime dependencies (cross-platform) - set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl openblas gfortran) - # On Windows, also include dl.dll (dlfcn-win32 POSIX emulation layer) - if(WIN32) - list(APPEND MLX_INCLUDE_REGEXES "^dl\\.dll$") - endif() - - # Keep mlx/mlxc targets separate from runtime dependencies so - # --strip only applies to the binaries we build, not vendor DLLs/libs. - install(TARGETS mlx mlxc - RUNTIME_DEPENDENCY_SET mlx_runtime_deps - RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - FRAMEWORK DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - ) - install(RUNTIME_DEPENDENCY_SET mlx_runtime_deps - DIRECTORIES ${MLX_RUNTIME_DIRS} - PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES} - PRE_EXCLUDE_REGEXES ".*" - RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR - LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR - ) - - if(TARGET jaccl) - install(TARGETS jaccl - RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - FRAMEWORK DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX - ) - endif() - - # Install the Metal library for macOS arm64 (must be colocated with the binary) - # Metal backend is only built for arm64, not x86_64 - if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - install(FILES ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/backend/metal/kernels/mlx.metallib - DESTINATION ${OLLAMA_INSTALL_DIR} - COMPONENT MLX) - endif() - - # Install headers for NVRTC JIT compilation at runtime. - # MLX's own install rules use the default component so they get skipped by - # --component MLX. Headers are installed alongside libmlx in OLLAMA_INSTALL_DIR. - # - # Layout: - # ${OLLAMA_INSTALL_DIR}/include/cccl/{cuda,nv}/ — CCCL headers - # ${OLLAMA_INSTALL_DIR}/include/*.h — CUDA toolkit headers - # - # MLX's jit_module.cpp resolves CCCL via - # current_binary_dir()[.parent_path()] / "include" / "cccl" - # On Linux, MLX's jit_module.cpp resolves CCCL via - # current_binary_dir().parent_path() / "include" / "cccl", so we create a - # symlink from lib/ollama/include -> ${OLLAMA_RUNNER_DIR}/include - # This will need refinement if we add multiple CUDA versions for MLX in the future. - # CUDA runtime headers are found via CUDA_PATH env var (set by mlxrunner). - if(EXISTS ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/cuda) - install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/cuda - DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl - COMPONENT MLX) - install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/nv - DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl - COMPONENT MLX) - endif() - - # Install minimal CUDA toolkit headers needed by MLX JIT kernels. - # These are the transitive closure of includes from mlx/backend/cuda/device/*.cuh. - # The Go mlxrunner sets CUDA_PATH to OLLAMA_INSTALL_DIR so MLX finds them at - # $CUDA_PATH/include/*.h via NVRTC --include-path. - if(CUDAToolkit_FOUND) - # CUDAToolkit_INCLUDE_DIRS may be a semicolon-separated list - # (e.g. ".../include;.../include/cccl"). Find the entry that - # contains the CUDA runtime headers we need. - set(_cuda_inc "") - foreach(_dir ${CUDAToolkit_INCLUDE_DIRS}) - if(EXISTS "${_dir}/cuda_runtime_api.h") - set(_cuda_inc "${_dir}") - break() - endif() - endforeach() - if(NOT _cuda_inc) - message(WARNING "Could not find cuda_runtime_api.h in CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}") - else() - set(_dst "${OLLAMA_INSTALL_DIR}/include") - set(_MLX_JIT_CUDA_HEADERS - builtin_types.h - cooperative_groups.h - cuda_bf16.h - cuda_bf16.hpp - cuda_device_runtime_api.h - cuda_fp16.h - cuda_fp16.hpp - cuda_fp8.h - cuda_fp8.hpp - cuda_runtime_api.h - device_types.h - driver_types.h - math_constants.h - surface_types.h - texture_types.h - vector_functions.h - vector_functions.hpp - vector_types.h - ) - foreach(_hdr ${_MLX_JIT_CUDA_HEADERS}) - install(FILES "${_cuda_inc}/${_hdr}" - DESTINATION ${_dst} - COMPONENT MLX) - endforeach() - # Subdirectory headers - install(DIRECTORY "${_cuda_inc}/cooperative_groups" - DESTINATION ${_dst} - COMPONENT MLX - FILES_MATCHING PATTERN "*.h") - install(FILES "${_cuda_inc}/crt/host_defines.h" - DESTINATION "${_dst}/crt" - COMPONENT MLX) - if(NOT WIN32 AND NOT APPLE) - install(CODE " - set(_link \"${CMAKE_INSTALL_PREFIX}/lib/ollama/include\") - set(_target \"${OLLAMA_RUNNER_DIR}/include\") - if(NOT EXISTS \${_link}) - execute_process(COMMAND \${CMAKE_COMMAND} -E create_symlink \${_target} \${_link}) - endif() - " COMPONENT MLX) - endif() - endif() - endif() - - # On Windows, explicitly install dl.dll (dlfcn-win32 POSIX dlopen emulation) - # RUNTIME_DEPENDENCIES auto-excludes it via POST_EXCLUDE_FILES_STRICT because - # dlfcn-win32 is a known CMake target with its own install rules (which install - # to the wrong destination). We must install it explicitly here. - if(WIN32) - install(FILES ${OLLAMA_BUILD_DIR}/dl.dll - DESTINATION ${OLLAMA_INSTALL_DIR} - COMPONENT MLX) - endif() - - # Manually install CUDA runtime libraries that MLX loads via dlopen - # (not detected by RUNTIME_DEPENDENCIES since they aren't link-time deps) - if(CUDAToolkit_FOUND) - file(GLOB MLX_CUDA_LIBS - "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*" - "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*" - "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*" - "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*" - "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*" - "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*" - "${CUDAToolkit_LIBRARY_DIR}/libcudnn.so*") - if(MLX_CUDA_LIBS) - install(FILES ${MLX_CUDA_LIBS} - DESTINATION ${OLLAMA_INSTALL_DIR} - COMPONENT MLX_VENDOR) - endif() - endif() + add_subdirectory(cmake/mlx) endif() diff --git a/Dockerfile b/Dockerfile index 37e7ac5f3..a7fb90c1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -187,6 +187,7 @@ ENV LAPACK_INCLUDE_DIRS=/usr/include/openblas ENV CGO_LDFLAGS="-L/usr/local/cuda-13/lib64 -L/usr/local/cuda-13/targets/x86_64-linux/lib/stubs" WORKDIR /go/src/github.com/ollama/ollama COPY CMakeLists.txt CMakePresets.json . +COPY cmake cmake COPY x/imagegen/mlx x/imagegen/mlx COPY go.mod go.sum . COPY MLX_VERSION MLX_C_VERSION . diff --git a/cmake/local.cmake b/cmake/local.cmake new file mode 100644 index 000000000..59a658233 --- /dev/null +++ b/cmake/local.cmake @@ -0,0 +1,301 @@ +# Local Ollama superbuild targets. +# +# This file keeps the repository-root CMake project focused on orchestration: +# it builds a runnable local Ollama payload by delegating llama.cpp work to the +# llama/server CMake project and building the Go binary into a matching layout. + +include(ExternalProject) + +set(OLLAMA_LLAMA_SERVER_BACKENDS "" CACHE STRING + "Semicolon-separated llama-server GPU backends to build: cuda-v12;cuda-v13;cuda-v13-windows;rocm;rocm-windows;vulkan;jetpack5;jetpack6") +set(OLLAMA_VERSION "0.0.0" CACHE STRING "Ollama version embedded in the local Go binary") + +string(REGEX REPLACE "^v" "" OLLAMA_VERSION "${OLLAMA_VERSION}") + +set(OLLAMA_LLAMA_SERVER_CONFIG_ARG) +if(CMAKE_CONFIGURATION_TYPES) + set(OLLAMA_LLAMA_SERVER_CONFIG_ARG --config Release) +endif() + +set(OLLAMA_LLAMA_SERVER_EXTERNAL_OPTIONS) +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) + list(APPEND OLLAMA_LLAMA_SERVER_EXTERNAL_OPTIONS BUILD_JOB_SERVER_AWARE TRUE) +endif() + +function(ollama_escape_cmake_list input output) + string(REPLACE ";" "|" _escaped "${input}") + set(${output} "${_escaped}" PARENT_SCOPE) +endfunction() + +function(ollama_add_llama_server_build name) + cmake_parse_arguments(ARG "" "RUNNER_DIR" "TARGETS;CMAKE_ARGS" ${ARGN}) + if(NOT ARG_TARGETS) + message(FATAL_ERROR "ollama_add_llama_server_build(${name}) requires TARGETS") + endif() + + if(WIN32 AND name STREQUAL "vulkan") + # The Vulkan shader generator nests deeply enough to hit Windows MAX_PATH. + set(_build_dir ${CMAKE_BINARY_DIR}/ls-vk) + else() + set(_build_dir ${CMAKE_BINARY_DIR}/llama-server-${name}) + endif() + set(_cmake_args + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR} + -DOLLAMA_RUNNER_DIR=${ARG_RUNNER_DIR} + -DGGML_NATIVE=OFF + -DGGML_OPENMP=OFF + ${ARG_CMAKE_ARGS} + ) + + if(APPLE) + if(CMAKE_OSX_ARCHITECTURES) + list(APPEND _cmake_args + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}) + endif() + if(CMAKE_OSX_DEPLOYMENT_TARGET) + list(APPEND _cmake_args + -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}) + endif() + endif() + if(DEFINED FETCHCONTENT_SOURCE_DIR_LLAMA_CPP) + list(APPEND _cmake_args + -DFETCHCONTENT_SOURCE_DIR_LLAMA_CPP=${FETCHCONTENT_SOURCE_DIR_LLAMA_CPP}) + endif() + + ExternalProject_Add(ollama-llama-server-${name} + SOURCE_DIR ${CMAKE_SOURCE_DIR}/llama/server + BINARY_DIR ${_build_dir} + CMAKE_ARGS ${_cmake_args} + BUILD_COMMAND ${CMAKE_COMMAND} --build + ${OLLAMA_LLAMA_SERVER_CONFIG_ARG} + --target ${ARG_TARGETS} + INSTALL_COMMAND ${CMAKE_COMMAND} --install + ${OLLAMA_LLAMA_SERVER_CONFIG_ARG} + --component llama-server + LIST_SEPARATOR | + BUILD_ALWAYS TRUE + ${OLLAMA_LLAMA_SERVER_EXTERNAL_OPTIONS} + USES_TERMINAL_CONFIGURE TRUE + USES_TERMINAL_BUILD TRUE + USES_TERMINAL_INSTALL TRUE) +endfunction() + +find_program(GO_EXECUTABLE go) + +if(NOT OLLAMA_GO_OUTPUT) + if(WIN32) + set(OLLAMA_GO_OUTPUT ${CMAKE_SOURCE_DIR}/ollama.exe) + else() + set(OLLAMA_GO_OUTPUT ${CMAKE_SOURCE_DIR}/ollama) + endif() +endif() +if(NOT IS_ABSOLUTE "${OLLAMA_GO_OUTPUT}") + set(OLLAMA_GO_OUTPUT "${CMAKE_SOURCE_DIR}/${OLLAMA_GO_OUTPUT}") +endif() +get_filename_component(OLLAMA_GO_OUTPUT "${OLLAMA_GO_OUTPUT}" ABSOLUTE) +set(OLLAMA_GO_OUTPUT "${OLLAMA_GO_OUTPUT}" CACHE FILEPATH "Output path for the local Ollama Go binary") +get_filename_component(OLLAMA_GO_OUTPUT_DIR "${OLLAMA_GO_OUTPUT}" DIRECTORY) + +set(OLLAMA_GO_LDFLAGS + "-s -w -X=github.com/ollama/ollama/version.Version=${OLLAMA_VERSION} -X=github.com/ollama/ollama/server.mode=release") +if(GO_EXECUTABLE) + add_custom_target(ollama-go ALL + COMMAND ${CMAKE_COMMAND} -E make_directory "${OLLAMA_GO_OUTPUT_DIR}" + COMMAND ${CMAKE_COMMAND} -E env CGO_ENABLED=1 + ${GO_EXECUTABLE} build -trimpath -ldflags "${OLLAMA_GO_LDFLAGS}" -o "${OLLAMA_GO_OUTPUT}" . + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + BYPRODUCTS ${OLLAMA_GO_OUTPUT} + COMMENT "Building Ollama Go binary" + VERBATIM) +else() + add_custom_target(ollama-go ALL + COMMAND ${CMAKE_COMMAND} -E echo + "Go executable not found. Install Go or set GO_EXECUTABLE to build the local Ollama binary." + COMMAND ${CMAKE_COMMAND} -E false + COMMENT "Building Ollama Go binary" + VERBATIM) +endif() + +set(_cpu_args) +if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + list(APPEND _cpu_args + -DBUILD_SHARED_LIBS=OFF + -DGGML_BACKEND_DL=OFF + -DGGML_METAL=ON + -DGGML_METAL_EMBED_LIBRARY=ON) +else() + list(APPEND _cpu_args + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON) + if(APPLE) + list(APPEND _cpu_args -DGGML_METAL=OFF) + endif() +endif() + +ollama_add_llama_server_build(local + RUNNER_DIR "" + TARGETS llama-server llama-quantize + CMAKE_ARGS ${_cpu_args}) + +add_custom_target(ollama-local ALL + DEPENDS ollama-go ollama-llama-server-local + COMMENT "Building local Ollama payload") + +install(PROGRAMS "${OLLAMA_GO_OUTPUT}" + DESTINATION "${CMAKE_INSTALL_BINDIR}" + COMPONENT ollama-local) + +set(_backend_targets) +foreach(_backend IN LISTS OLLAMA_LLAMA_SERVER_BACKENDS) + if(_backend STREQUAL "cuda-v12") + set(_cuda_arch "${CMAKE_CUDA_ARCHITECTURES}") + if(NOT _cuda_arch) + set(_cuda_arch "50-virtual;52-virtual;60-virtual;61-virtual;70;75;80;86;89;90;90a;120") + endif() + ollama_escape_cmake_list("${_cuda_arch}" _cuda_arch_arg) + set(_cuda_flags "${CMAKE_CUDA_FLAGS}") + if(NOT _cuda_flags) + set(_cuda_flags "-Wno-deprecated-gpu-targets -t 2") + endif() + ollama_add_llama_server_build(cuda-v12 + RUNNER_DIR cuda_v12 + TARGETS ggml-cuda + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CUDA=ON + -DCMAKE_CUDA_ARCHITECTURES=${_cuda_arch_arg} + -DCMAKE_CUDA_FLAGS=${_cuda_flags} + -DOLLAMA_GPU_BACKEND=cuda) + list(APPEND _backend_targets ollama-llama-server-cuda-v12) + elseif(_backend STREQUAL "cuda-v13") + set(_cuda_arch "${CMAKE_CUDA_ARCHITECTURES}") + if(NOT _cuda_arch) + set(_cuda_arch "75-virtual;80-virtual;86-virtual;89-virtual;90-virtual;90a-virtual;100-virtual;103-virtual;110-virtual;120-virtual;121-virtual") + endif() + ollama_escape_cmake_list("${_cuda_arch}" _cuda_arch_arg) + set(_cuda_flags "${CMAKE_CUDA_FLAGS}") + if(NOT _cuda_flags) + set(_cuda_flags "-t 4") + endif() + ollama_add_llama_server_build(cuda-v13 + RUNNER_DIR cuda_v13 + TARGETS ggml-cuda + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CUDA=ON + -DCMAKE_CUDA_ARCHITECTURES=${_cuda_arch_arg} + -DCMAKE_CUDA_FLAGS=${_cuda_flags} + -DOLLAMA_GPU_BACKEND=cuda) + list(APPEND _backend_targets ollama-llama-server-cuda-v13) + elseif(_backend STREQUAL "cuda-v13-windows") + set(_cuda_arch "${CMAKE_CUDA_ARCHITECTURES}") + if(NOT _cuda_arch) + set(_cuda_arch "75-virtual;89-virtual;100-virtual;120-virtual") + endif() + ollama_escape_cmake_list("${_cuda_arch}" _cuda_arch_arg) + set(_cuda_flags "${CMAKE_CUDA_FLAGS}") + if(NOT _cuda_flags) + set(_cuda_flags "-t 4") + endif() + ollama_add_llama_server_build(cuda-v13-windows + RUNNER_DIR cuda_v13 + TARGETS ggml-cuda + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CUDA=ON + -DCMAKE_CUDA_ARCHITECTURES=${_cuda_arch_arg} + -DCMAKE_CUDA_FLAGS=${_cuda_flags} + -DOLLAMA_GPU_BACKEND=cuda) + list(APPEND _backend_targets ollama-llama-server-cuda-v13-windows) + elseif(_backend STREQUAL "rocm" OR _backend STREQUAL "rocm-windows") + set(_amd_targets "${AMDGPU_TARGETS}") + if(NOT _amd_targets) + set(_amd_targets "${CMAKE_HIP_ARCHITECTURES}") + endif() + if(NOT _amd_targets) + if(_backend STREQUAL "rocm-windows") + set(_amd_targets "gfx942;gfx950;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1200;gfx1201") + else() + set(_amd_targets "gfx942;gfx950;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1200;gfx1201;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-") + endif() + endif() + ollama_escape_cmake_list("${_amd_targets}" _amd_targets_arg) + set(_hip_flags "${CMAKE_HIP_FLAGS}") + if(NOT _hip_flags) + set(_hip_flags "-parallel-jobs=4") + endif() + set(_rocm_args + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_HIP=ON + -DCMAKE_HIP_PLATFORM=amd + -DAMDGPU_TARGETS=${_amd_targets_arg} + -DCMAKE_HIP_FLAGS=${_hip_flags} + -DOLLAMA_GPU_BACKEND=hip) + if(CMAKE_PREFIX_PATH) + ollama_escape_cmake_list("${CMAKE_PREFIX_PATH}" _prefix_path_arg) + list(APPEND _rocm_args -DCMAKE_PREFIX_PATH=${_prefix_path_arg}) + endif() + if(_backend STREQUAL "rocm-windows") + list(APPEND _rocm_args + "-DCMAKE_C_FLAGS=-parallel-jobs=4 -Wno-ignored-attributes -Wno-deprecated-pragma" + "-DCMAKE_CXX_FLAGS=-parallel-jobs=4 -Wno-ignored-attributes -Wno-deprecated-pragma") + endif() + ollama_add_llama_server_build(${_backend} + RUNNER_DIR rocm + TARGETS ggml-hip + CMAKE_ARGS ${_rocm_args}) + list(APPEND _backend_targets ollama-llama-server-${_backend}) + elseif(_backend STREQUAL "vulkan") + ollama_add_llama_server_build(vulkan + RUNNER_DIR vulkan + TARGETS ggml-vulkan + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_VULKAN=ON + -DOLLAMA_GPU_BACKEND=vulkan) + list(APPEND _backend_targets ollama-llama-server-vulkan) + elseif(_backend STREQUAL "jetpack5") + ollama_add_llama_server_build(jetpack5 + RUNNER_DIR cuda_jetpack5 + TARGETS ggml-cuda + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CUDA=ON + -DCMAKE_CUDA_ARCHITECTURES=72\;87 + -DOLLAMA_GPU_BACKEND=cuda) + list(APPEND _backend_targets ollama-llama-server-jetpack5) + elseif(_backend STREQUAL "jetpack6") + ollama_add_llama_server_build(jetpack6 + RUNNER_DIR cuda_jetpack6 + TARGETS ggml-cuda + CMAKE_ARGS + -DBUILD_SHARED_LIBS=ON + -DGGML_BACKEND_DL=ON + -DGGML_CUDA=ON + -DCMAKE_CUDA_ARCHITECTURES=87 + -DOLLAMA_GPU_BACKEND=cuda) + list(APPEND _backend_targets ollama-llama-server-jetpack6) + else() + message(FATAL_ERROR + "Unknown OLLAMA_LLAMA_SERVER_BACKENDS entry '${_backend}'") + endif() +endforeach() + +if(_backend_targets) + add_custom_target(ollama-llama-server-backends ALL + DEPENDS ${_backend_targets} + COMMENT "Building llama-server GPU backends") +endif() + +install(DIRECTORY "${CMAKE_BINARY_DIR}/lib/ollama/" + DESTINATION "lib/ollama" + COMPONENT ollama-local + USE_SOURCE_PERMISSIONS) diff --git a/cmake/mlx/CMakeLists.txt b/cmake/mlx/CMakeLists.txt new file mode 100644 index 000000000..018682dbe --- /dev/null +++ b/cmake/mlx/CMakeLists.txt @@ -0,0 +1,189 @@ +# Ollama MLX runtime build and install rules. +# +# The repository root CMake project is evolving into an orchestration layer. +# Keep MLX-specific target setup here so the root can later coordinate MLX, +# llama-server, and Go builds without carrying backend-specific install logic. + +if(MLX_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES) + check_language(CUDA) +endif() + +message(STATUS "Setting up MLX (this takes a while...)") +add_subdirectory(${CMAKE_SOURCE_DIR}/x/imagegen/mlx ${CMAKE_BINARY_DIR}/x/imagegen/mlx) + +# Find CUDA toolkit if MLX is built with CUDA support. +find_package(CUDAToolkit) + +# Build list of directories for runtime dependency resolution. +set(MLX_RUNTIME_DIRS ${CUDAToolkit_BIN_DIR} ${CUDAToolkit_BIN_DIR}/x64 ${CUDAToolkit_LIBRARY_DIR}) +# Add cuDNN bin paths for DLLs (Windows MLX CUDA builds). +# CUDNN_ROOT_DIR is the standard CMake variable for cuDNN location. +if(DEFINED ENV{CUDNN_ROOT_DIR}) + # cuDNN 9.x has versioned subdirectories under bin/ (e.g., bin/13.0/). + file(GLOB CUDNN_BIN_SUBDIRS "$ENV{CUDNN_ROOT_DIR}/bin/*") + list(APPEND MLX_RUNTIME_DIRS ${CUDNN_BIN_SUBDIRS}) +endif() +# Add build output directory and MLX dependency build directories. +list(APPEND MLX_RUNTIME_DIRS ${OLLAMA_BUILD_DIR}) +# OpenBLAS DLL location (pre-built zip extracts into openblas-src/bin/). +list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/openblas-src/bin) +# NCCL: on Linux, if real NCCL is found, cmake bundles libnccl.so via the +# regex below. If NCCL is not found, MLX links a static stub (OBJECT lib) +# so there is no runtime dependency. This path covers the stub build dir +# for windows so we include the DLL in our dependencies. +list(APPEND MLX_RUNTIME_DIRS ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/distributed/nccl/nccl_stub-prefix/src/nccl_stub-build/Release) + +# Base regexes for runtime dependencies (cross-platform). +set(MLX_INCLUDE_REGEXES cublas cublasLt cudart cufft nvrtc nvrtc-builtins cudnn nccl openblas gfortran) +# On Windows, also include dl.dll (dlfcn-win32 POSIX emulation layer). +if(WIN32) + list(APPEND MLX_INCLUDE_REGEXES "^dl\\.dll$") +endif() + +# Keep mlx/mlxc targets separate from runtime dependencies so --strip only +# applies to the binaries we build, not vendor DLLs/libs. +install(TARGETS mlx mlxc + RUNTIME_DEPENDENCY_SET mlx_runtime_deps + RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX + LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX + FRAMEWORK DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX +) +install(RUNTIME_DEPENDENCY_SET mlx_runtime_deps + DIRECTORIES ${MLX_RUNTIME_DIRS} + PRE_INCLUDE_REGEXES ${MLX_INCLUDE_REGEXES} + PRE_EXCLUDE_REGEXES ".*" + RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR + LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX_VENDOR +) + +if(TARGET jaccl) + install(TARGETS jaccl + RUNTIME DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX + LIBRARY DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX + FRAMEWORK DESTINATION ${OLLAMA_INSTALL_DIR} COMPONENT MLX + ) +endif() + +# Install the Metal library for macOS arm64 (must be colocated with the binary). +# Metal backend is only built for arm64, not x86_64. +if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") + install(FILES ${CMAKE_BINARY_DIR}/_deps/mlx-build/mlx/backend/metal/kernels/mlx.metallib + DESTINATION ${OLLAMA_INSTALL_DIR} + COMPONENT MLX) +endif() + +# Install headers for NVRTC JIT compilation at runtime. +# MLX's own install rules use the default component so they get skipped by +# --component MLX. Headers are installed alongside libmlx in OLLAMA_INSTALL_DIR. +# +# Layout: +# ${OLLAMA_INSTALL_DIR}/include/cccl/{cuda,nv}/ - CCCL headers +# ${OLLAMA_INSTALL_DIR}/include/*.h - CUDA toolkit headers +# +# MLX's jit_module.cpp resolves CCCL via +# current_binary_dir()[.parent_path()] / "include" / "cccl" +# On Linux, MLX's jit_module.cpp resolves CCCL via +# current_binary_dir().parent_path() / "include" / "cccl", so we create a +# symlink from lib/ollama/include -> ${OLLAMA_RUNNER_DIR}/include. +# This will need refinement if we add multiple CUDA versions for MLX in the future. +# CUDA runtime headers are found via CUDA_PATH env var (set by mlxrunner). +if(EXISTS ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/cuda) + install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/cuda + DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl + COMPONENT MLX) + install(DIRECTORY ${CMAKE_BINARY_DIR}/_deps/cccl-src/include/nv + DESTINATION ${OLLAMA_INSTALL_DIR}/include/cccl + COMPONENT MLX) +endif() + +# Install minimal CUDA toolkit headers needed by MLX JIT kernels. +# These are the transitive closure of includes from mlx/backend/cuda/device/*.cuh. +# The Go mlxrunner sets CUDA_PATH to OLLAMA_INSTALL_DIR so MLX finds them at +# $CUDA_PATH/include/*.h via NVRTC --include-path. +if(CUDAToolkit_FOUND) + # CUDAToolkit_INCLUDE_DIRS may be a semicolon-separated list + # (e.g. ".../include;.../include/cccl"). Find the entry that + # contains the CUDA runtime headers we need. + set(_cuda_inc "") + foreach(_dir ${CUDAToolkit_INCLUDE_DIRS}) + if(EXISTS "${_dir}/cuda_runtime_api.h") + set(_cuda_inc "${_dir}") + break() + endif() + endforeach() + if(NOT _cuda_inc) + message(WARNING "Could not find cuda_runtime_api.h in CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}") + else() + set(_dst "${OLLAMA_INSTALL_DIR}/include") + set(_MLX_JIT_CUDA_HEADERS + builtin_types.h + cooperative_groups.h + cuda_bf16.h + cuda_bf16.hpp + cuda_device_runtime_api.h + cuda_fp16.h + cuda_fp16.hpp + cuda_fp8.h + cuda_fp8.hpp + cuda_runtime_api.h + device_types.h + driver_types.h + math_constants.h + surface_types.h + texture_types.h + vector_functions.h + vector_functions.hpp + vector_types.h + ) + foreach(_hdr ${_MLX_JIT_CUDA_HEADERS}) + install(FILES "${_cuda_inc}/${_hdr}" + DESTINATION ${_dst} + COMPONENT MLX) + endforeach() + # Subdirectory headers. + install(DIRECTORY "${_cuda_inc}/cooperative_groups" + DESTINATION ${_dst} + COMPONENT MLX + FILES_MATCHING PATTERN "*.h") + install(FILES "${_cuda_inc}/crt/host_defines.h" + DESTINATION "${_dst}/crt" + COMPONENT MLX) + if(NOT WIN32 AND NOT APPLE) + install(CODE " + set(_link \"${CMAKE_INSTALL_PREFIX}/lib/ollama/include\") + set(_target \"${OLLAMA_RUNNER_DIR}/include\") + if(NOT EXISTS \${_link}) + execute_process(COMMAND \${CMAKE_COMMAND} -E create_symlink \${_target} \${_link}) + endif() + " COMPONENT MLX) + endif() + endif() +endif() + +# On Windows, explicitly install dl.dll (dlfcn-win32 POSIX dlopen emulation). +# RUNTIME_DEPENDENCIES auto-excludes it via POST_EXCLUDE_FILES_STRICT because +# dlfcn-win32 is a known CMake target with its own install rules (which install +# to the wrong destination). We must install it explicitly here. +if(WIN32) + install(FILES ${OLLAMA_BUILD_DIR}/dl.dll + DESTINATION ${OLLAMA_INSTALL_DIR} + COMPONENT MLX) +endif() + +# Manually install CUDA runtime libraries that MLX loads via dlopen +# (not detected by RUNTIME_DEPENDENCIES since they aren't link-time deps). +if(CUDAToolkit_FOUND) + file(GLOB MLX_CUDA_LIBS + "${CUDAToolkit_LIBRARY_DIR}/libcudart.so*" + "${CUDAToolkit_LIBRARY_DIR}/libcublas.so*" + "${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so*" + "${CUDAToolkit_LIBRARY_DIR}/libnvrtc.so*" + "${CUDAToolkit_LIBRARY_DIR}/libnvrtc-builtins.so*" + "${CUDAToolkit_LIBRARY_DIR}/libcufft.so*" + "${CUDAToolkit_LIBRARY_DIR}/libcudnn.so*") + if(MLX_CUDA_LIBS) + install(FILES ${MLX_CUDA_LIBS} + DESTINATION ${OLLAMA_INSTALL_DIR} + COMPONENT MLX_VENDOR) + endif() +endif() diff --git a/docs/development.md b/docs/development.md index 9aee1607d..a2c8311c3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -3,9 +3,11 @@ Install prerequisites: - [Go](https://go.dev/doc/install) -- C/C++ Compiler e.g. Clang on macOS, [TDM-GCC](https://github.com/jmeubank/tdm-gcc/releases/latest) (Windows amd64) or [llvm-mingw](https://github.com/mstorsjo/llvm-mingw) (Windows arm64), GCC/Clang on Linux. +- [CMake](https://cmake.org/download/) 3.24 or newer +- C/C++ compiler: Clang on macOS, Visual Studio 2022 C++ tools on Windows, or GCC/Clang on Linux +- [Ninja](https://github.com/ninja-build/ninja/releases) in `PATH` is recommended, especially on Windows -Then build and run Ollama from the root directory of the repository: +For pure Go iteration against an existing native payload, run Ollama from the repository root: ```shell go run . serve @@ -14,10 +16,44 @@ go run . serve > [!NOTE] > Ollama includes native code compiled with CGO. From time to time these data structures can change and CGO can get out of sync resulting in unexpected crashes. You can force a full build of the native code by running `go clean -cache` first. +## Native build model + +For a fresh checkout, or after changing native code, build from the repository root. Use an explicit job count instead of bare `--parallel`; increase `4` only if the machine has enough CPU and memory headroom. + +```shell +cmake -B build . +cmake --build build --parallel 4 +./ollama serve +``` + +To build `llama-server` GPU backends through the same root build, select the backend and target explicitly: + +```shell +cmake -B build-gpu . -DOLLAMA_LLAMA_SERVER_BACKENDS=vulkan +cmake --build build-gpu --target ollama-llama-server-vulkan --parallel 4 +``` + +Supported backend values are `cuda-v12`, `cuda-v13`, `cuda-v13-windows`, `rocm`, `rocm-windows`, `vulkan`, `jetpack5`, and `jetpack6`. + +Use standard CMake architecture overrides to narrow GPU builds for local hardware: + +```shell +# CUDA +cmake -B build-gpu . -DOLLAMA_LLAMA_SERVER_BACKENDS=cuda-v13 -DCMAKE_CUDA_ARCHITECTURES=native + +# ROCm / HIP +cmake -B build-gpu . -DOLLAMA_LLAMA_SERVER_BACKENDS=rocm -DCMAKE_HIP_ARCHITECTURES=gfx1100 +``` + +`AMDGPU_TARGETS` is also accepted for ROCm when matching llama.cpp-specific target strings is necessary. ## macOS (Apple Silicon) -macOS Apple Silicon supports Metal which is built-in to the Ollama binary. No additional steps are required. +macOS Apple Silicon supports Metal for local native builds. For a release-style payload: + +```shell +./scripts/build_darwin.sh -a arm64 +``` ## macOS (Intel) @@ -25,41 +61,37 @@ Install prerequisites: - [CMake](https://cmake.org/download/) or `brew install cmake` -Then, configure and build the project: +Then build the Darwin payload: ```shell -cmake -B build -cmake --build build +./scripts/build_darwin.sh -a amd64 ``` Lastly, run Ollama: ```shell -go run . serve +dist/darwin-amd64/ollama serve ``` ## Windows Install prerequisites: -- [CMake](https://cmake.org/download/) -- [Ninja](https://github.com/ninja-build/ninja/releases) in `PATH` - [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) including the Native Desktop Workload - (Optional) AMD GPU support - [ROCm](https://rocm.docs.amd.com/en/latest/) - (Optional) NVIDIA GPU support - - [CUDA SDK](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=11&target_type=exe_network) -- (Optional) VULKAN GPU support - - [VULKAN SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs + - [CUDA SDK](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_type=exe_network) +- (Optional) Vulkan GPU support + - [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs - (Optional) MLX engine support - [CUDA 13+ SDK](https://developer.nvidia.com/cuda-downloads) - [cuDNN 9+](https://developer.nvidia.com/cudnn) -Then, configure and build the project: +Then build a minimal CPU payload and Go binary: -```shell -cmake -B build -cmake --build build --config Release +```powershell +.\scripts\build_windows.ps1 cpu ollama ``` > Building for Vulkan requires VULKAN_SDK environment variable: @@ -74,35 +106,30 @@ cmake --build build --config Release > ``` > [!IMPORTANT] -> Building for ROCm requires additional flags: -> ``` -> cmake -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -> cmake --build build --config Release -> ``` - - +> Prefer the build script for release-style GPU payloads. It wires the platform-specific compiler, SDK, and install layout details. Lastly, run Ollama: -```shell -go run . serve +```powershell +.\dist\windows-amd64\ollama.exe serve ``` +For native CMake iteration, use the repository-root CMake build shown in [Native build model](#native-build-model). Ninja is recommended when available. + ## Windows (ARM) -Windows ARM does not support additional acceleration libraries at this time. Do not use cmake, simply `go run` or `go build`. +Windows ARM does not support additional acceleration libraries at this time. The Windows build script can cross-compile the CPU llama-server payload when the ARM64 cross-compile toolchain is installed; otherwise it skips that payload for local developer builds. ## Linux Install prerequisites: -- [CMake](https://cmake.org/download/) or `sudo apt install cmake` or `sudo dnf install cmake` - (Optional) AMD GPU support - [ROCm](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/install/quick-start.html) - (Optional) NVIDIA GPU support - [CUDA SDK](https://developer.nvidia.com/cuda-downloads) -- (Optional) VULKAN GPU support - - [VULKAN SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs +- (Optional) Vulkan GPU support + - [Vulkan SDK](https://vulkan.lunarg.com/sdk/home) - useful for AMD/Intel GPUs - Or install via package manager: `sudo apt install vulkan-sdk` (Ubuntu/Debian) or `sudo dnf install vulkan-sdk` (Fedora/CentOS) - (Optional) MLX engine support - [CUDA 13+ SDK](https://developer.nvidia.com/cuda-downloads) @@ -112,22 +139,17 @@ Install prerequisites: > Ensure prerequisites are in `PATH` before running CMake. -Then, configure and build the project: +For a release-style Linux payload, use the Docker-backed build script: ```shell -cmake -B build -cmake --build build +./scripts/build_linux.sh ``` -Lastly, run Ollama: - -```shell -go run . serve -``` +For native CMake iteration, use the repository-root CMake build shown in [Native build model](#native-build-model). ## MLX Engine (Optional) -The MLX engine enables running safetensor based models. It requires building the [MLX](https://github.com/ml-explore/mlx) and [MLX-C](https://github.com/ml-explore/mlx-c) shared libraries separately via CMake. On MacOS, MLX leverages the Metal library to run on the GPU, and on Windows and Linux, runs on NVIDIA GPUs via CUDA v13. +The MLX engine enables running safetensor based models. It requires building the [MLX](https://github.com/ml-explore/mlx) and [MLX-C](https://github.com/ml-explore/mlx-c) shared libraries via the repository-root CMake presets. The root project delegates MLX-specific rules to `cmake/mlx`. On macOS, MLX leverages the Metal library to run on the GPU, and on Windows and Linux, runs on NVIDIA GPUs via CUDA v13. ### macOS (Apple Silicon) @@ -146,9 +168,10 @@ xcrun metal Then build: ```shell -cmake -B build --preset MLX -cmake --build build --preset MLX --parallel +cmake --preset MLX +cmake --build --preset MLX --parallel 4 cmake --install build --component MLX +cmake --install build --component MLX_VENDOR ``` > [!NOTE] @@ -159,9 +182,10 @@ cmake --install build --component MLX Requires CUDA 13+ and [cuDNN](https://developer.nvidia.com/cudnn) 9+. ```shell -cmake -B build --preset "MLX CUDA 13" -cmake --build build --target mlx --target mlxc --config Release --parallel +cmake --preset "MLX CUDA 13" +cmake --build --preset "MLX CUDA 13" --parallel 4 cmake --install build --component MLX --strip +cmake --install build --component MLX_VENDOR ``` ### Local MLX source overrides @@ -183,7 +207,7 @@ OLLAMA_MLX_SOURCE=../mlx OLLAMA_MLX_C_SOURCE=../mlx-c ./scripts/build_darwin.sh ```powershell $env:OLLAMA_MLX_SOURCE="../mlx" $env:OLLAMA_MLX_C_SOURCE="../mlx-c" -./scripts/build_darwin.ps1 +./scripts/build_windows.ps1 ``` ## Docker @@ -208,11 +232,11 @@ go test ./... ## Library detection -Ollama looks for acceleration libraries in the following paths relative to the `ollama` executable: +Ollama looks for native helper binaries and acceleration libraries in installed and local development layouts: -* `./lib/ollama` (Windows) -* `../lib/ollama` (Linux) -* `.` (macOS) -* `build/lib/ollama` (for development) +* `../lib/ollama` for standard installs where `ollama` is under `bin/` +* `./lib/ollama` for Windows release-style payloads and local dist output +* `.` for macOS release artifacts that colocate helpers with `ollama` +* `build/lib/ollama` and `dist//lib/ollama` for local development builds If the libraries are not found, Ollama will not run with any acceleration libraries. diff --git a/llama/server/CMakePresets.json b/llama/server/CMakePresets.json index 2aa931ec0..b7d97f9a5 100644 --- a/llama/server/CMakePresets.json +++ b/llama/server/CMakePresets.json @@ -151,12 +151,12 @@ { "name": "cpu", "configurePreset": "cpu", - "targets": ["llama-server", "llama-quantize", "ggml-cpu"] + "targets": ["llama-server", "llama-quantize"] }, { "name": "cpu-arm64", "configurePreset": "cpu-arm64", - "targets": ["llama-server", "llama-quantize", "ggml-cpu"] + "targets": ["llama-server", "llama-quantize"] }, { "name": "darwin", diff --git a/llm/llama_binary.go b/llm/llama_binary.go index 71ca6f371..c263f6624 100644 --- a/llm/llama_binary.go +++ b/llm/llama_binary.go @@ -94,12 +94,16 @@ func llamaCppBinaryCandidates(name string, search llamaCppBinarySearch) []string add(base) // Per-architecture local dist output keeps helpers under lib/ollama. add(filepath.Join(base, "lib", "ollama")) + // Standard CMake installs put ollama in bin/ and helpers in ../lib/ollama/. + add(filepath.Join(base, "..", "lib", "ollama")) case "linux": // Linux packages install ollama in bin/ and helpers in ../lib/ollama/. add(filepath.Join(base, "..", "lib", "ollama")) case "windows": // Windows packages keep ollama.exe at top level with lib/ as a peer. add(filepath.Join(base, "lib", "ollama")) + // Standard CMake installs put ollama.exe in bin/ and helpers in ../lib/ollama/. + add(filepath.Join(base, "..", "lib", "ollama")) default: add(filepath.Join(base, "lib", "ollama")) add(filepath.Join(base, "..", "lib", "ollama")) diff --git a/llm/llama_binary_test.go b/llm/llama_binary_test.go index a866dc682..a6fccb9b5 100644 --- a/llm/llama_binary_test.go +++ b/llm/llama_binary_test.go @@ -12,9 +12,10 @@ func TestLlamaCppBinaryCandidates(t *testing.T) { root := t.TempDir() tests := []struct { - name string - search llamaCppBinarySearch - want []string + name string + search llamaCppBinarySearch + want []string + wantFirst string }{ { name: "linux production layout", @@ -24,6 +25,7 @@ func TestLlamaCppBinaryCandidates(t *testing.T) { goarch: "amd64", }, want: []string{filepath.Join(root, "linux", "lib", "ollama", "llama-server")}, + wantFirst: filepath.Join(root, "linux", "lib", "ollama", "llama-server"), }, { name: "windows production layout", @@ -33,6 +35,7 @@ func TestLlamaCppBinaryCandidates(t *testing.T) { goarch: "amd64", }, want: []string{filepath.Join(root, "windows", "lib", "ollama", "llama-server.exe")}, + wantFirst: filepath.Join(root, "windows", "lib", "ollama", "llama-server.exe"), }, { name: "darwin production layout", @@ -42,6 +45,25 @@ func TestLlamaCppBinaryCandidates(t *testing.T) { goarch: "arm64", }, want: []string{filepath.Join(root, "Ollama.app", "Contents", "Resources", "llama-server")}, + wantFirst: filepath.Join(root, "Ollama.app", "Contents", "Resources", "llama-server"), + }, + { + name: "darwin standard install layout", + search: llamaCppBinarySearch{ + executable: filepath.Join(root, "darwin", "bin", "ollama"), + goos: "darwin", + goarch: "arm64", + }, + want: []string{filepath.Join(root, "darwin", "lib", "ollama", "llama-server")}, + }, + { + name: "windows standard install layout", + search: llamaCppBinarySearch{ + executable: filepath.Join(root, "windows", "bin", "ollama.exe"), + goos: "windows", + goarch: "amd64", + }, + want: []string{filepath.Join(root, "windows", "lib", "ollama", "llama-server.exe")}, }, { name: "local per-architecture dist layout", @@ -109,6 +131,9 @@ func TestLlamaCppBinaryCandidates(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { candidates := llamaCppBinaryCandidates("llama-server", tt.search) + if tt.wantFirst != "" && candidates[0] != tt.wantFirst { + t.Fatalf("first candidate = %q, want %q; all candidates: %v", candidates[0], tt.wantFirst, candidates) + } if tt.search.libOllamaPath != "" && candidates[0] != tt.want[0] { t.Fatalf("first candidate = %q, want %q; all candidates: %v", candidates[0], tt.want[0], candidates) } diff --git a/ml/path.go b/ml/path.go index 88f4f6f88..d8c05f1f7 100644 --- a/ml/path.go +++ b/ml/path.go @@ -6,56 +6,118 @@ import ( "runtime" ) -// LibPath is a path to lookup dynamic libraries -// in development it's usually 'build/lib/ollama' -// in distribution builds it's 'lib/ollama' on Windows -// '../lib/ollama' on Linux and the executable's directory on macOS -// note: distribution builds, additional GPU-specific libraries are -// found in subdirectories of the returned path, such as -// 'cuda_v12', 'rocm', etc. -var LibOllamaPath string = func() string { +type libOllamaPathSearch struct { + executable string + workingDir string + goos string + goarch string +} + +// LibOllamaPath is the root used to find bundled llama.cpp and MLX runtime +// libraries. GPU-specific libraries live in backend subdirectories such as +// cuda_v12, rocm, vulkan, and mlx_cuda_v13. +var LibOllamaPath = func() string { exe, err := os.Executable() if err != nil { return "" } - if eval, err := filepath.EvalSymlinks(exe); err == nil { exe = eval } - var libPath string - switch runtime.GOOS { - case "windows": - libPath = filepath.Join(filepath.Dir(exe), "lib", "ollama") - case "linux": - libPath = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama") - case "darwin": - libPath = filepath.Dir(exe) - } - cwd, err := os.Getwd() if err != nil { - return "" + cwd = "" } - distPath := filepath.Join("dist", runtime.GOOS+"-"+runtime.GOARCH, "lib", "ollama") - paths := []string{ - libPath, + return findLibOllamaPath(libOllamaPathSearch{ + executable: exe, + workingDir: cwd, + goos: runtime.GOOS, + goarch: runtime.GOARCH, + }) +}() - // release build paths for local development - filepath.Join(filepath.Dir(exe), distPath), - filepath.Join(cwd, distPath), - - // build paths for development - filepath.Join(filepath.Dir(exe), "build", "lib", "ollama"), - filepath.Join(cwd, "build", "lib", "ollama"), - } - - for _, p := range paths { - if _, err := os.Stat(p); err == nil { - return p +func findLibOllamaPath(search libOllamaPathSearch) string { + candidates := libOllamaPathCandidates(search) + for _, path := range candidates { + if libOllamaPathExists(path) { + return path } } - return filepath.Dir(exe) -}() + if search.executable != "" { + return filepath.Dir(search.executable) + } + return "" +} + +func libOllamaPathCandidates(search libOllamaPathSearch) []string { + goos := search.goos + if goos == "" { + goos = runtime.GOOS + } + goarch := search.goarch + if goarch == "" { + goarch = runtime.GOARCH + } + + seen := map[string]bool{} + var candidates []string + add := func(path string) { + if path == "" { + return + } + path = filepath.Clean(path) + if !seen[path] { + seen[path] = true + candidates = append(candidates, path) + } + } + + if search.executable != "" { + exeDir := filepath.Dir(search.executable) + switch goos { + case "darwin": + // Local dist output and standard installs keep helpers under lib/ollama. + add(filepath.Join(exeDir, "lib", "ollama")) + add(filepath.Join(exeDir, "..", "lib", "ollama")) + case "linux": + add(filepath.Join(exeDir, "..", "lib", "ollama")) + add(filepath.Join(exeDir, "lib", "ollama")) + case "windows": + add(filepath.Join(exeDir, "lib", "ollama")) + add(filepath.Join(exeDir, "..", "lib", "ollama")) + default: + add(filepath.Join(exeDir, "lib", "ollama")) + add(filepath.Join(exeDir, "..", "lib", "ollama")) + } + addLocalLibOllamaPaths(add, exeDir, goos, goarch) + if goos == "darwin" { + // macOS release artifacts colocate native helpers with ollama. + add(exeDir) + } + } + addLocalLibOllamaPaths(add, search.workingDir, goos, goarch) + + return candidates +} + +func addLocalLibOllamaPaths(add func(string), base, goos, goarch string) { + if base == "" { + return + } + add(filepath.Join(base, "build", "lib", "ollama")) + add(filepath.Join(base, "dist", goos+"-"+goarch, "lib", "ollama")) + if goos+"_"+goarch != goos+"-"+goarch { + add(filepath.Join(base, "dist", goos+"_"+goarch, "lib", "ollama")) + } + if goos == "darwin" { + add(filepath.Join(base, "dist", "darwin")) + } +} + +func libOllamaPathExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/ml/path_test.go b/ml/path_test.go new file mode 100644 index 000000000..52256b80c --- /dev/null +++ b/ml/path_test.go @@ -0,0 +1,116 @@ +package ml + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFindLibOllamaPath(t *testing.T) { + root := t.TempDir() + + tests := []struct { + name string + search libOllamaPathSearch + dirs []string + want string + }{ + { + name: "darwin release layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "darwin-app", "Ollama.app", "Contents", "Resources", "ollama"), + goos: "darwin", + goarch: "arm64", + }, + dirs: []string{filepath.Join(root, "darwin-app", "Ollama.app", "Contents", "Resources")}, + want: filepath.Join(root, "darwin-app", "Ollama.app", "Contents", "Resources"), + }, + { + name: "darwin standard install layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "darwin-install", "bin", "ollama"), + goos: "darwin", + goarch: "arm64", + }, + dirs: []string{filepath.Join(root, "darwin-install", "lib", "ollama")}, + want: filepath.Join(root, "darwin-install", "lib", "ollama"), + }, + { + name: "windows release layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "windows-release", "ollama.exe"), + goos: "windows", + goarch: "amd64", + }, + dirs: []string{filepath.Join(root, "windows-release", "lib", "ollama")}, + want: filepath.Join(root, "windows-release", "lib", "ollama"), + }, + { + name: "windows standard install layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "windows-install", "bin", "ollama.exe"), + goos: "windows", + goarch: "amd64", + }, + dirs: []string{filepath.Join(root, "windows-install", "lib", "ollama")}, + want: filepath.Join(root, "windows-install", "lib", "ollama"), + }, + { + name: "linux standard install layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "linux-install", "bin", "ollama"), + goos: "linux", + goarch: "amd64", + }, + dirs: []string{filepath.Join(root, "linux-install", "lib", "ollama")}, + want: filepath.Join(root, "linux-install", "lib", "ollama"), + }, + { + name: "local linux underscore dist layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "linux-dev", "ollama"), + workingDir: filepath.Join(root, "linux-dev"), + goos: "linux", + goarch: "amd64", + }, + dirs: []string{filepath.Join(root, "linux-dev", "dist", "linux_amd64", "lib", "ollama")}, + want: filepath.Join(root, "linux-dev", "dist", "linux_amd64", "lib", "ollama"), + }, + { + name: "mlx-only standard install layout", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "mlx-install", "bin", "ollama"), + goos: "linux", + goarch: "amd64", + }, + dirs: []string{filepath.Join(root, "mlx-install", "lib", "ollama")}, + want: filepath.Join(root, "mlx-install", "lib", "ollama"), + }, + { + name: "darwin local build layout before executable directory fallback", + search: libOllamaPathSearch{ + executable: filepath.Join(root, "darwin-dev", "ollama"), + workingDir: filepath.Join(root, "darwin-dev"), + goos: "darwin", + goarch: "arm64", + }, + dirs: []string{filepath.Join(root, "darwin-dev", "build", "lib", "ollama")}, + want: filepath.Join(root, "darwin-dev", "build", "lib", "ollama"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, dir := range tt.dirs { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + + got := findLibOllamaPath(tt.search) + if got != tt.want { + t.Fatalf("findLibOllamaPath() = %q, want %q; candidates: %v", got, tt.want, libOllamaPathCandidates(tt.search)) + } + }) + } +} diff --git a/x/imagegen/mlx/mlx.go b/x/imagegen/mlx/mlx.go index 9b8f6d5bd..bf953b0a6 100644 --- a/x/imagegen/mlx/mlx.go +++ b/x/imagegen/mlx/mlx.go @@ -1719,6 +1719,28 @@ func mlxLibName() string { } } +func findMLXLibraryInDir(dir, libName string) string { + if dir == "" { + return "" + } + + candidate := filepath.Join(dir, libName) + if _, err := os.Stat(candidate); err == nil { + return candidate + } + + if mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx*")); err == nil { + for _, mlxDir := range mlxDirs { + candidate = filepath.Join(mlxDir, libName) + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + } + + return "" +} + // findMLXLibrary searches for the MLX shared library in standard locations. // Returns the path to the library, or empty string if not found. func findMLXLibrary() string { @@ -1727,18 +1749,9 @@ func findMLXLibrary() string { // 1. OLLAMA_LIBRARY_PATH — check each dir and mlx_* subdirs if paths, ok := os.LookupEnv("OLLAMA_LIBRARY_PATH"); ok { for _, dir := range filepath.SplitList(paths) { - candidate := filepath.Join(dir, libName) - if _, err := os.Stat(candidate); err == nil { + if candidate := findMLXLibraryInDir(dir, libName); candidate != "" { return candidate } - if mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx*")); err == nil { - for _, mlxDir := range mlxDirs { - candidate = filepath.Join(mlxDir, libName) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - } - } } } @@ -1750,8 +1763,7 @@ func findMLXLibrary() string { exeDir := filepath.Dir(exe) // Check exe dir directly (macOS copies dylib here) - candidate := filepath.Join(exeDir, libName) - if _, err := os.Stat(candidate); err == nil { + if candidate := findMLXLibraryInDir(exeDir, libName); candidate != "" { return candidate } @@ -1761,22 +1773,27 @@ func findMLXLibrary() string { filepath.Join(exeDir, "lib", "ollama"), filepath.Join(exeDir, "..", "lib", "ollama"), } { - if mlxDirs, err := filepath.Glob(filepath.Join(libOllamaDir, "mlx*")); err == nil { - for _, mlxDir := range mlxDirs { - candidate = filepath.Join(mlxDir, libName) - if _, err := os.Stat(candidate); err == nil { - return candidate - } - } + if candidate := findMLXLibraryInDir(libOllamaDir, libName); candidate != "" { + return candidate } } } // 3. Build directory (for tests run from repo root) if cwd, err := os.Getwd(); err == nil { - candidate := filepath.Join(cwd, "build", "lib", "ollama", libName) - if _, err := os.Stat(candidate); err == nil { - return candidate + for _, dir := range []string{ + filepath.Join(cwd, "build", "lib", "ollama"), + filepath.Join(cwd, "dist", runtime.GOOS+"-"+runtime.GOARCH, "lib", "ollama"), + filepath.Join(cwd, "dist", runtime.GOOS+"_"+runtime.GOARCH, "lib", "ollama"), + } { + if candidate := findMLXLibraryInDir(dir, libName); candidate != "" { + return candidate + } + } + if runtime.GOOS == "darwin" { + if candidate := findMLXLibraryInDir(filepath.Join(cwd, "dist", "darwin"), libName); candidate != "" { + return candidate + } } } diff --git a/x/mlxrunner/mlx/dynamic.go b/x/mlxrunner/mlx/dynamic.go index e4b74c953..33e8e1f82 100644 --- a/x/mlxrunner/mlx/dynamic.go +++ b/x/mlxrunner/mlx/dynamic.go @@ -70,7 +70,8 @@ func tryLoadFromDir(dir string) bool { } // libOllamaRoots returns candidate directories for MLX dynamic libraries. -// Production: exe_dir/lib/ollama (dist tarball) and exe_dir (app bundle). +// Production: exe_dir/lib/ollama (Windows release layout), +// exe_dir/../lib/ollama (standard bin/lib layout), and exe_dir (macOS bundle). // Development: build/lib/ollama and build/*/lib/ollama. func libOllamaRoots() []string { var roots []string @@ -84,11 +85,13 @@ func libOllamaRoots() []string { switch runtime.GOOS { case "darwin": roots = append(roots, filepath.Join(exeDir, "lib", "ollama")) + roots = append(roots, filepath.Join(exeDir, "..", "lib", "ollama")) roots = append(roots, exeDir) // app bundle: Contents/Resources/ case "linux": roots = append(roots, filepath.Join(exeDir, "..", "lib", "ollama")) case "windows": roots = append(roots, filepath.Join(exeDir, "lib", "ollama")) + roots = append(roots, filepath.Join(exeDir, "..", "lib", "ollama")) } }