Merge tag 'v0.3.7' into ampere-build-3.0.0

Author: dkupnicki

.github/workflows/build-and-release.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2019, macos-12]
+        os: [ubuntu-20.04, windows-2019, macos-13]
 
     steps:
       - uses: actions/checkout@v4
@@ -42,7 +42,7 @@ jobs:
         shell: cmd
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.1
+        uses: pypa/cibuildwheel@v2.22.0
         env:
           # disable repair
           CIBW_REPAIR_WHEEL_COMMAND: ""
@@ -69,7 +69,7 @@ jobs:
           platforms: linux/arm64
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.1
+        uses: pypa/cibuildwheel@v2.22.0
         env:
           CIBW_SKIP: "*musllinux* pp*"
           CIBW_REPAIR_WHEEL_COMMAND: ""

.github/workflows/build-wheels-cuda.yaml

@@ -22,7 +22,7 @@ jobs:
           $matrix = @{
               'os' = @('ubuntu-latest', 'windows-2019')
               'pyver' = @("3.9", "3.10", "3.11", "3.12")
-              'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.0")
+              'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1")
               'releasetag' = @("basic")
           }
 
@@ -59,13 +59,11 @@ jobs:
           cache: 'pip'
 
       - name: Setup Mamba
-        uses: conda-incubator/setup-miniconda@v3.0.4
+        uses: conda-incubator/setup-miniconda@v3.1.0
         with:
-          activate-environment: "build"
+          activate-environment: "llamacpp"
           python-version: ${{ matrix.pyver }}
-          miniforge-variant: Mambaforge
           miniforge-version: latest
-          use-mamba: true
           add-pip-as-python-dependency: true
           auto-activate-base: false
 

.github/workflows/build-wheels-metal.yaml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-12, macos-13, macos-14]
+        os: [macos-13, macos-14, macos-15]
 
     steps:
       - uses: actions/checkout@v4
@@ -43,7 +43,7 @@ jobs:
         shell: cmd
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.1
+        uses: pypa/cibuildwheel@v2.22.0
         env:
           # disable repair
           CIBW_REPAIR_WHEEL_COMMAND: ""

.github/workflows/generate-index-from-release.yaml

@@ -44,7 +44,8 @@ jobs:
           ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'
           ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$'
           ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$'
+          # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
+          # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
           ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$'
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/test.yaml

@@ -51,19 +51,11 @@ jobs:
           path: ~/.cache/huggingface/hub
           key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
       - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
         run: |
           python -m pip install --upgrade pip
           python -m pip install uv
           python -m uv pip install -e .[all] --verbose
         shell: bash
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: cmd
       - name: Test with pytest
         run: |
           python -m pytest
@@ -90,30 +82,21 @@ jobs:
         with:
           path: ~/.cache/huggingface/hub
           key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-          
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: bash
 
       - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
         run: |
           python -m pip install --upgrade pip
           python -m pip install uv
           python -m uv pip install -e .[all] --verbose
-        shell: cmd     
+        shell: cmd
 
       - name: Test with pytest
         run: |
           python -m pytest
 
   build-macos:
     needs: download-model
-    runs-on: macos-latest
+    runs-on: macos-13
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
@@ -128,35 +111,33 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
 
+      - name: System Info
+        run: |
+          uname -a
+          sysctl -n machdep.cpu.brand_string
+          python3 -c "import platform; print(platform.machine(), platform.architecture())"
+
       - name: Restore model cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/huggingface/hub
           key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
 
       - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
+          python3 -m pip install --upgrade pip
+          python3 -m pip install uv
+          python3 -m uv pip install -e .[all] --verbose
+          CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose
         shell: bash
 
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: cmd
-          
       - name: Test with pytest
         run: |
-          python -m pytest
+          python3 -m pytest
 
   build-macos-metal:
     needs: download-model
-    runs-on: macos-latest
+    runs-on: macos-13
     steps:
       - uses: actions/checkout@v4
         with:
@@ -167,25 +148,24 @@ jobs:
         with:
           python-version: "3.9"
 
+      - name: System Info
+        run: |
+          uname -a
+          sysctl -n machdep.cpu.brand_string
+          python3 -c "import platform; print(platform.machine(), platform.architecture())"
+
       - name: Restore model cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/huggingface/hub
           key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
 
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
+      - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose
+          python3 -m pip install --upgrade pip
+          CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose
         shell: bash
 
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose
       - name: Test with pytest
         run: |
-          python -m pytest
+          python3 -m pytest

CHANGELOG.md

@@ -7,6 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.7]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@794fe23f29fb40104975c91fe19f23798f7c726e
+- fix(ci): Fix the CUDA workflow by @oobabooga in #1894
+- fix: error showing time spent in llama perf context print, adds `no_perf` flag to `Llama` class by @shakalaca in #1898
+
+## [0.3.6]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@f7cd13301c2a88f97073fd119072b4cc92c08df1
+- fix(server): streaming resource lock by @gjpower in #1879
+
+## [0.3.5]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@26a8406ba9198eb6fdd8329fa717555b4f77f05f
+- fix(ci): Fix release by updating macos runner image to non-deprecated version by @abetlen in afedfc888462f9a6e809dc9455eb3b663764cc3f
+- fix(server): add missing await statements for async exit_stack handling by @gjpower in #1858
+
+## [0.3.4]
+
+- fix(ci): Build wheels for macos 13-15, cuda 12.1-12.4 by @abetlen in ca808028bd16b8327bd84128d48015a4b1304690
+
+## [0.3.3]
+
+- feat: Update llama.cpp to ggerganov/llama.cpp@ce8784bdb153ff7794dde5a50b0ebfa51baa6171
+- fix: chat API logprobs format by @domdomegg in #1788
+- feat: Add support for CUDA 12.6, fix CUDA 12.5 by @Smartappli in #1775
+- fix: Make content not required in ChatCompletionRequestAssistantMessage by @feloy in #1807
+- fix: Fix pickling of Llama class by setting seed from _seed member by @abetlen in 2523472c3eccb9ab9277117cc4ff705212b6888a
+- fix: Fix logit-bias type hint by @ddh0 in #1802
+- fix(server): Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context by @gjpower in #1798
+- fix(server): Added missing exit_stack.close() to /v1/chat/completions by @Ian321 in #1796
+- fix(examples): Refactor Batching notebook to use new sampler chain API by @lukestanley in #1793
+- fix(docs): Update development instructions by @Florents-Tselai in #1833
+- fix(docs): Remove ref to llama_eval in llama_cpp.py docs by @richdougherty in #1819
+
 ## [0.3.2]
 
 - feat: Update llama.cpp to ggerganov/llama.cpp@74d73dc85cc2057446bf63cc37ff649ae7cebd80

CMakeLists.txt

@@ -62,18 +62,35 @@ if (LLAMA_BUILD)
     # Enable building of the common library
     set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE)
 
-    # Building llama
-    if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64")
-        # Need to disable these llama.cpp flags on Apple x86_64,
-        # otherwise users may encounter invalid instruction errors
-        set(GGML_AVX "Off" CACHE BOOL "ggml: enable AVX" FORCE)
-        set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE)
-        set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE)
-        set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE)
-    endif()
-
+    # Architecture detection and settings for Apple platforms
     if (APPLE)
-        set(GGML_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE)
+        # Get the target architecture
+        execute_process(
+            COMMAND uname -m
+            OUTPUT_VARIABLE HOST_ARCH
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+
+        # If CMAKE_OSX_ARCHITECTURES is not set, use the host architecture
+        if(NOT CMAKE_OSX_ARCHITECTURES)
+            set(CMAKE_OSX_ARCHITECTURES ${HOST_ARCH} CACHE STRING "Build architecture for macOS" FORCE)
+        endif()
+
+        message(STATUS "Host architecture: ${HOST_ARCH}")
+        message(STATUS "Target architecture: ${CMAKE_OSX_ARCHITECTURES}")
+
+        # Configure based on target architecture
+        if(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64")
+            # Intel Mac settings
+            set(GGML_AVX "OFF" CACHE BOOL "ggml: enable AVX" FORCE)
+            set(GGML_AVX2 "OFF" CACHE BOOL "ggml: enable AVX2" FORCE)
+            set(GGML_FMA "OFF" CACHE BOOL "ggml: enable FMA" FORCE)
+            set(GGML_F16C "OFF" CACHE BOOL "ggml: enable F16C" FORCE)
+        endif()
+
+        # Metal settings (enable for both architectures)
+        set(GGML_METAL "ON" CACHE BOOL "ggml: enable Metal" FORCE)
+        set(GGML_METAL_EMBED_LIBRARY "ON" CACHE BOOL "ggml: embed metal library" FORCE)
     endif()
 
     add_subdirectory(vendor/llama.cpp)
@@ -128,7 +145,7 @@ if (LLAMA_BUILD)
         # Building llava
         add_subdirectory(vendor/llama.cpp/examples/llava)
         set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava")
-        # Set CUDA_ARCHITECTURES to OFF on Windows
+
         if (WIN32)
             set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF)
         endif()

Makefile

@@ -21,6 +21,15 @@ build.debug:
 		--config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-ggdb -O0';-DCMAKE_CXX_FLAGS='-ggdb -O0'" \
 		--editable .
 
+build.debug.extra:
+	python3 -m pip install \
+		--verbose \
+		--config-settings=cmake.verbose=true \
+		--config-settings=logging.level=INFO \
+		--config-settings=install.strip=false  \
+		--config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-fsanitize=address -ggdb -O0';-DCMAKE_CXX_FLAGS='-fsanitize=address -ggdb -O0'" \
+		--editable .
+
 build.cuda:
 	CMAKE_ARGS="-DGGML_CUDA=on" python3 -m pip install --verbose -e .
 
@@ -46,7 +55,7 @@ build.rpc:
 	CMAKE_ARGS="-DGGML_RPC=on" python3 -m pip install --verbose -e .
 
 build.sdist:
-	python3 -m build --sdist
+	python3 -m build --sdist --verbose
 
 deploy.pypi:
 	python3 -m twine upload dist/*
@@ -56,23 +65,23 @@ deploy.gh-docs:
 	mkdocs gh-deploy
 
 test:
-	python3 -m pytest
+	python3 -m pytest --full-trace -v
 
 docker:
 	docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile .
 
 run-server:
-	uvicorn --factory llama.server:app --host ${HOST} --port ${PORT}
+	python3 -m llama_cpp.server --model ${MODEL}
 
 clean:
 	- cd vendor/llama.cpp && make clean
 	- cd vendor/llama.cpp && rm libllama.so
 	- rm -rf _skbuild
-	- rm llama_cpp/*.so
-	- rm llama_cpp/*.dylib
-	- rm llama_cpp/*.metal
-	- rm llama_cpp/*.dll
-	- rm llama_cpp/*.lib
+	- rm llama_cpp/lib/*.so
+	- rm llama_cpp/lib/*.dylib
+	- rm llama_cpp/lib/*.metal
+	- rm llama_cpp/lib/*.dll
+	- rm llama_cpp/lib/*.lib
 
 .PHONY: \
 	update \