xgrammar: add standalone CMake project for native library (#18611)

Move the ollama_xgrammar target into mlxrunner/xgrammar/native so it
can be configured on its own against an installed xgrammar. cmake/mlx
now adds it as a subdirectory and still uses the pinned xgrammar.

.github/scripts/prepare_mlx_darwin.sh

@@ -52,8 +52,6 @@ read_pin() {
   tr -d '[:space:]' <"$1"
 }
 
-# Native wrapper sources compiled into libollama_xgrammar.dylib — keep in
-# sync with the ollama_xgrammar target in cmake/mlx/CMakeLists.txt.
 xgrammar_native_dir=mlxrunner/xgrammar/native
 
 # Payload build rules beyond the MLX_VERSION/MLX_C_VERSION pins.

.github/workflows/test-xgrammar.yaml

@@ -0,0 +1,75 @@
+name: test-xgrammar
+
+on:
+  pull_request:
+    paths:
+      - 'cmake/mlx/CMakeLists.txt'
+      - 'mlxrunner/xgrammar/**'
+      - '.github/workflows/test-xgrammar.yaml'
+
+permissions:
+  contents: read
+
+jobs:
+  standalone:
+    runs-on: macos-latest
+    env:
+      CGO_ENABLED: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Cache MLX Darwin release payload
+        uses: actions/cache@v4
+        with:
+          path: .cache/mlx-darwin-release
+          key: mlx-darwin-${{ hashFiles('MLX_VERSION', 'MLX_C_VERSION', 'cmake/local.cmake', 'cmake/apply-git-patches.cmake', 'cmake/mlx/CMakeLists.txt', 'cmake/mlx/CMakePresets.json', 'mlx/CMakeLists.txt', 'mlx/compat/**', 'mlxrunner/xgrammar/native/**') }}
+      - name: Prepare MLX Darwin release payload
+        run: .github/scripts/prepare_mlx_darwin.sh
+      - name: Install pinned xgrammar into a prefix
+        run: |
+          set -euo pipefail
+          version="$(sed -n 's/^set(XGRAMMAR_VERSION \(.*\))$/\1/p' cmake/mlx/CMakeLists.txt)"
+          [ -n "${version}" ] || { echo "XGRAMMAR_VERSION not found in cmake/mlx/CMakeLists.txt" >&2; exit 1; }
+          echo "XGRAMMAR_VERSION=${version}" >>"${GITHUB_ENV}"
+          src="${RUNNER_TEMP}/xgrammar-src"
+          build="${RUNNER_TEMP}/xgrammar-build"
+          git clone --depth 1 --branch "${version}" --recurse-submodules --shallow-submodules \
+            https://github.com/mlc-ai/xgrammar.git "${src}"
+          # xgrammar's cmake/config.cmake overrides -D options unless the build dir has its own.
+          mkdir -p "${build}"
+          echo 'set(XGRAMMAR_BUILD_PYTHON_BINDINGS OFF)' >"${build}/config.cmake"
+          cmake -S "${src}" -B "${build}" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${RUNNER_TEMP}/xgrammar-prefix"
+          cmake --build "${build}" --parallel
+          cmake --install "${build}"
+      - name: Build xgrammar library against installed xgrammar
+        run: |
+          set -euo pipefail
+          cmake -S mlxrunner/xgrammar/native -B build/xgrammar-standalone \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_PREFIX_PATH="${RUNNER_TEMP}/xgrammar-prefix" \
+            -DXGRAMMAR_VERSION="${XGRAMMAR_VERSION}"
+          cmake --build build/xgrammar-standalone
+      - name: Verify only the ollama_xgrammar API is exported
+        run: |
+          set -euo pipefail
+          lib=build/xgrammar-standalone/libollama_xgrammar.dylib
+          if nm -gU "${lib}" | awk '{print $3}' | grep -v '^_ollama_xgrammar_'; then
+            echo "unexpected exported symbols in ${lib}" >&2
+            exit 1
+          fi
+      - name: Replace the payload's xgrammar library
+        run: |
+          set -euo pipefail
+          for variant in build/lib/ollama/mlx_metal_v*; do
+            cp -f build/xgrammar-standalone/libollama_xgrammar.dylib "${variant}/"
+          done
+      - name: go test
+        run: |
+          set -euo pipefail
+          # The tests skip when the library is missing; treat a skip as failure.
+          go test -count=1 -v ./mlxrunner/xgrammar/... | tee "${RUNNER_TEMP}/xgrammar-test.log"
+          if grep -- '--- SKIP' "${RUNNER_TEMP}/xgrammar-test.log"; then
+            exit 1
+          fi

cmake/mlx/CMakeLists.txt

@@ -87,18 +87,7 @@ target_compile_definitions(xgrammar PUBLIC
     XGRAMMAR_ENABLE_CPPTRACE=0
     XGRAMMAR_ENABLE_INTERNAL_CHECK=0)
 
-add_library(ollama_xgrammar SHARED
-    "${OLLAMA_SOURCE_DIR}/mlxrunner/xgrammar/native/xgrammar.cpp")
-target_include_directories(ollama_xgrammar PRIVATE
-    "${OLLAMA_SOURCE_DIR}/mlxrunner/xgrammar/native")
-target_compile_definitions(ollama_xgrammar PRIVATE
-    OLLAMA_XGRAMMAR_BUILD=1
-    OLLAMA_XGRAMMAR_VERSION="${XGRAMMAR_VERSION}")
-# Export only the ollama_xgrammar_* API; xgrammar's own C++ symbols stay hidden.
-set_target_properties(ollama_xgrammar PROPERTIES
-    CXX_VISIBILITY_PRESET hidden
-    VISIBILITY_INLINES_HIDDEN ON)
-target_link_libraries(ollama_xgrammar PRIVATE xgrammar Threads::Threads)
+add_subdirectory(${OLLAMA_SOURCE_DIR}/mlxrunner/xgrammar/native ${CMAKE_BINARY_DIR}/xgrammar)
 
 # Find CUDA toolkit if MLX is built with CUDA support.
 find_package(CUDAToolkit)

mlxrunner/xgrammar/native/CMakeLists.txt

@@ -0,0 +1,42 @@
+cmake_minimum_required(VERSION 3.24)
+
+project(OllamaXGrammar CXX)
+
+# cmake/mlx provides the pinned xgrammar target and XGRAMMAR_VERSION.
+if(NOT TARGET xgrammar)
+    set(XGRAMMAR_VERSION "unknown" CACHE STRING "XGrammar release reported by the library")
+    find_path(XGRAMMAR_INCLUDE_DIR xgrammar/xgrammar.h REQUIRED)
+    find_path(DLPACK_INCLUDE_DIR dlpack/dlpack.h REQUIRED)
+    find_library(XGRAMMAR_LIBRARY xgrammar REQUIRED)
+    add_library(xgrammar UNKNOWN IMPORTED)
+    set_target_properties(xgrammar PROPERTIES
+        IMPORTED_LOCATION "${XGRAMMAR_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${XGRAMMAR_INCLUDE_DIR};${DLPACK_INCLUDE_DIR}")
+endif()
+
+find_package(Threads REQUIRED)
+
+add_library(ollama_xgrammar SHARED xgrammar.cpp)
+target_compile_features(ollama_xgrammar PRIVATE cxx_std_17)
+target_include_directories(ollama_xgrammar PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
+target_compile_definitions(ollama_xgrammar PRIVATE
+    OLLAMA_XGRAMMAR_BUILD=1
+    OLLAMA_XGRAMMAR_VERSION="${XGRAMMAR_VERSION}")
+# Export only the ollama_xgrammar_* API; xgrammar's own C++ symbols stay hidden.
+set_target_properties(ollama_xgrammar PROPERTIES
+    CXX_VISIBILITY_PRESET hidden
+    VISIBILITY_INLINES_HIDDEN ON)
+if(APPLE)
+    target_link_options(ollama_xgrammar PRIVATE "LINKER:-exported_symbol,_ollama_xgrammar_*")
+elseif(UNIX)
+    target_link_options(ollama_xgrammar PRIVATE "LINKER:--exclude-libs,ALL")
+endif()
+target_link_libraries(ollama_xgrammar PRIVATE xgrammar Threads::Threads)
+
+# cmake/mlx installs the library with the rest of the MLX payload.
+if(PROJECT_IS_TOP_LEVEL)
+    include(GNUInstallDirs)
+    install(TARGETS ollama_xgrammar
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()