Skip to content

Commit 4132293

Browse files
author
Mug
committed
Merge branch 'main' of https://github.com/abetlen/llama-cpp-python into local-lib
2 parents 76131d5 + 241d608 commit 4132293

23 files changed

+1307
-42
lines changed

‎.github/workflows/build-and-release.yaml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: Build Release
2+
3+
on: workflow_dispatch
4+
5+
permissions:
6+
contents: write
7+
8+
jobs:
9+
build_wheels:
10+
name: Build wheels on ${{ matrix.os }}
11+
runs-on: ${{ matrix.os }}
12+
strategy:
13+
matrix:
14+
os: [ubuntu-latest, windows-latest, macOS-latest]
15+
16+
steps:
17+
- uses: actions/checkout@v3
18+
with:
19+
submodules: "true"
20+
21+
# Used to host cibuildwheel
22+
- uses: actions/setup-python@v3
23+
24+
- name: Install cibuildwheel
25+
run: python -m pip install cibuildwheel==2.12.1
26+
27+
- name: Install dependencies
28+
run: |
29+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
30+
31+
- name: Build wheels
32+
run: python -m cibuildwheel --output-dir wheelhouse
33+
34+
- uses: actions/upload-artifact@v3
35+
with:
36+
path: ./wheelhouse/*.whl
37+
38+
build_sdist:
39+
name: Build source distribution
40+
runs-on: ubuntu-latest
41+
42+
steps:
43+
- uses: actions/checkout@v3
44+
with:
45+
submodules: "true"
46+
- uses: actions/setup-python@v3
47+
- name: Install dependencies
48+
run: |
49+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
50+
- name: Build source distribution
51+
run: |
52+
python setup.py sdist
53+
- uses: actions/upload-artifact@v3
54+
with:
55+
path: ./dist/*.tar.gz
56+
57+
release:
58+
name: Release
59+
needs: [build_wheels, build_sdist]
60+
runs-on: ubuntu-latest
61+
62+
steps:
63+
- uses: actions/download-artifact@v3
64+
with:
65+
name: artifact
66+
path: dist
67+
- uses: softprops/action-gh-release@v1
68+
with:
69+
files: dist/*
70+
env:
71+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/publish-to-test.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
2+
3+
name: Publish to TestPyPI
4+
5+
on: workflow_dispatch
6+
7+
jobs:
8+
build-n-publish:
9+
name: Build and publish
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- uses: actions/checkout@v3
14+
with:
15+
submodules: "true"
16+
- name: Set up Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: "3.8"
20+
- name: Install dependencies
21+
run: |
22+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
23+
- name: Build source distribution
24+
run: |
25+
python setup.py sdist
26+
- name: Publish to Test PyPI
27+
uses: pypa/gh-action-pypi-publish@release/v1
28+
with:
29+
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
30+
repository-url: https://test.pypi.org/legacy/

‎.github/workflows/publish.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Publish to PyPI
2+
3+
# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
4+
5+
on: workflow_dispatch
6+
7+
jobs:
8+
build-n-publish:
9+
name: Build and publish
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- uses: actions/checkout@v3
14+
with:
15+
submodules: "true"
16+
- name: Set up Python
17+
uses: actions/setup-python@v4
18+
with:
19+
python-version: "3.8"
20+
- name: Install dependencies
21+
run: |
22+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
23+
- name: Build source distribution
24+
run: |
25+
python setup.py sdist
26+
- name: Publish distribution to PyPI
27+
# TODO: move to tag based releases
28+
# if: startsWith(github.ref, 'refs/tags')
29+
uses: pypa/gh-action-pypi-publish@release/v1
30+
with:
31+
password: ${{ secrets.PYPI_API_TOKEN }}

‎.github/workflows/test.yaml‎

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
name: Tests
22

33
on:
4+
pull_request:
5+
branches:
6+
- main
47
push:
58
branches:
69
- main
710

811
jobs:
9-
build:
12+
build-linux:
1013

1114
runs-on: ubuntu-latest
1215
strategy:
@@ -23,8 +26,54 @@ jobs:
2326
python-version: ${{ matrix.python-version }}
2427
- name: Install dependencies
2528
run: |
26-
python -m pip install --upgrade pip pytest cmake scikit-build
27-
python3 setup.py develop
29+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
30+
pip install . -v
2831
- name: Test with pytest
2932
run: |
3033
pytest
34+
35+
build-windows:
36+
37+
runs-on: windows-latest
38+
strategy:
39+
matrix:
40+
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
41+
42+
steps:
43+
- uses: actions/checkout@v3
44+
with:
45+
submodules: "true"
46+
- name: Set up Python ${{ matrix.python-version }}
47+
uses: actions/setup-python@v4
48+
with:
49+
python-version: ${{ matrix.python-version }}
50+
- name: Install dependencies
51+
run: |
52+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
53+
pip install . -v
54+
- name: Test with pytest
55+
run: |
56+
pytest
57+
58+
build-macos:
59+
60+
runs-on: macos-latest
61+
strategy:
62+
matrix:
63+
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
64+
65+
steps:
66+
- uses: actions/checkout@v3
67+
with:
68+
submodules: "true"
69+
- name: Set up Python ${{ matrix.python-version }}
70+
uses: actions/setup-python@v4
71+
with:
72+
python-version: ${{ matrix.python-version }}
73+
- name: Install dependencies
74+
run: |
75+
python -m pip install --upgrade pip pytest cmake scikit-build setuptools
76+
pip install . -v
77+
- name: Test with pytest
78+
run: |
79+
pytest

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,4 +163,4 @@ cython_debug/
163163
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
164164
# and can be added to the global gitignore or merged into this file. For a more nuclear
165165
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
166-
#.idea/
166+
.idea/

‎CMakeLists.txt‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,26 @@ cmake_minimum_required(VERSION 3.4...3.22)
22

33
project(llama_cpp)
44

5-
set(BUILD_SHARED_LIBS "On")
6-
7-
add_subdirectory(vendor/llama.cpp)
8-
9-
install(TARGETS llama LIBRARY DESTINATION llama_cpp)
5+
if (UNIX)
6+
add_custom_command(
7+
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so
8+
COMMAND make libllama.so
9+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp
10+
)
11+
add_custom_target(
12+
run ALL
13+
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so
14+
)
15+
install(
16+
FILES ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/libllama.so
17+
DESTINATION llama_cpp
18+
)
19+
else()
20+
set(BUILD_SHARED_LIBS "On")
21+
add_subdirectory(vendor/llama.cpp)
22+
install(
23+
TARGETS llama
24+
LIBRARY DESTINATION llama_cpp
25+
RUNTIME DESTINATION llama_cpp
26+
)
27+
endif(UNIX)

‎README.md‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,26 +15,26 @@ This package provides:
1515
- OpenAI-like API
1616
- LangChain compatibility
1717

18-
# Installation
18+
## Installation
1919

2020
Install from PyPI:
2121

2222
```bash
2323
pip install llama-cpp-python
2424
```
2525

26-
# Usage
26+
## High-level API
2727

2828
```python
2929
>>> from llama_cpp import Llama
30-
>>> llm = Llama(model_path="models/7B/...")
30+
>>> llm = Llama(model_path="./models/7B/ggml-model.bin")
3131
>>> output = llm("Q: Name the planets in the solar system? A: ", max_tokens=32, stop=["Q:", "\n"], echo=True)
3232
>>> print(output)
3333
{
3434
"id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
3535
"object": "text_completion",
3636
"created": 1679561337,
37-
"model": "models/7B/...",
37+
"model": "./models/7B/ggml-model.bin",
3838
"choices": [
3939
{
4040
"text": "Q: Name the planets in the solar system? A: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune and Pluto.",
@@ -51,6 +51,27 @@ pip install llama-cpp-python
5151
}
5252
```
5353

54+
## Web Server
55+
56+
`llama-cpp-python` offers a web server which aims to act as a drop-in replacement for the OpenAI API.
57+
This allows you to use llama.cpp compatible models with any OpenAI compatible client (language libraries, services, etc).
58+
59+
To install the server package and get started:
60+
61+
```bash
62+
pip install llama-cpp-python[server]
63+
export MODEL=./models/7B/ggml-model.bin
64+
python3 -m llama_cpp.server
65+
```
66+
67+
Navigate to [http://localhost:8000/docs](http://localhost:8000/docs) to see the OpenAPI documentation.
68+
69+
## Low-level API
70+
71+
The low-level API is a direct `ctypes` binding to the C API provided by `llama.cpp`.
72+
The entire API can be found in [llama_cpp/llama_cpp.py](https://github.com/abetlen/llama-cpp-python/blob/master/llama_cpp/llama_cpp.py) and should mirror [llama.h](https://github.com/ggerganov/llama.cpp/blob/master/llama.h).
73+
74+
5475
# Documentation
5576

5677
Documentation is available at [https://abetlen.github.io/llama-cpp-python](https://abetlen.github.io/llama-cpp-python).

‎docs/index.md‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
# 🦙 Python Bindings for `llama.cpp`
1+
# Getting Started
22

3+
## 🦙 Python Bindings for `llama.cpp`
4+
5+
[![Documentation](https://img.shields.io/badge/docs-passing-green.svg)](https://abetlen.github.io/llama-cpp-python)
6+
[![Tests](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml)
37
[![PyPI](https://img.shields.io/pypi/v/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/)
48
[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/)
59
[![PyPI - License](https://img.shields.io/pypi/l/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/)
@@ -21,18 +25,18 @@ Install from PyPI:
2125
pip install llama-cpp-python
2226
```
2327

24-
## Usage
28+
## High-level API
2529

2630
```python
2731
>>> from llama_cpp import Llama
28-
>>> llm = Llama(model_path="models/7B/...")
32+
>>> llm = Llama(model_path="./models/7B/ggml-model.bin")
2933
>>> output = llm("Q: Name the planets in the solar system? A: ", max_tokens=32, stop=["Q:", "\n"], echo=True)
3034
>>> print(output)
3135
{
3236
"id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
3337
"object": "text_completion",
3438
"created": 1679561337,
35-
"model": "models/7B/...",
39+
"model": "./models/7B/ggml-model.bin",
3640
"choices": [
3741
{
3842
"text": "Q: Name the planets in the solar system? A: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune and Pluto.",
@@ -49,8 +53,33 @@ pip install llama-cpp-python
4953
}
5054
```
5155

56+
## Web Server
57+
58+
`llama-cpp-python` offers a web server which aims to act as a drop-in replacement for the OpenAI API.
59+
This allows you to use llama.cpp compatible models with any OpenAI compatible client (language libraries, services, etc).
60+
61+
To install the server package and get started:
62+
63+
```bash
64+
pip install llama-cpp-python[server]
65+
export MODEL=./models/7B/ggml-model.bin
66+
python3 -m llama_cpp.server
67+
```
68+
69+
Navigate to [http://localhost:8000/docs](http://localhost:8000/docs) to see the OpenAPI documentation.
70+
71+
## Low-level API
72+
73+
The low-level API is a direct `ctypes` binding to the C API provided by `llama.cpp`.
74+
The entire API can be found in [llama_cpp/llama_cpp.py](https://github.com/abetlen/llama-cpp-python/blob/master/llama_cpp/llama_cpp.py) and should mirror [llama.h](https://github.com/ggerganov/llama.cpp/blob/master/llama.h).
75+
76+
5277
## Development
5378

79+
This package is under active development and I welcome any contributions.
80+
81+
To get started, clone the repository and install the package in development mode:
82+
5483
```bash
5584
git clone git@github.com:abetlen/llama-cpp-python.git
5685
git submodule update --init --recursive

0 commit comments

Comments
 (0)