Skip to content

Commit 77bb6c9

Browse files
authored
llamacpp wrong default value passed for f16_kv (langchain-ai#3320)
Fixes default f16_kv value in llamacpp; corrects incorrect parameter passed. See: https://github.com/abetlen/llama-cpp-python/blob/ba3959eafd38080f3bf3028746406f350a8ef793/llama_cpp/llama.py#L33 Fixes langchain-ai#3241 Fixes langchain-ai#3301
1 parent 3a1bdce commit 77bb6c9

File tree

1 file changed

+3
-3
lines changed

1 file changed

+3
-3
lines changed

langchain/llms/llamacpp.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,13 @@ class LlamaCpp(LLM):
3131
"""Token context window."""
3232

3333
n_parts: int = Field(-1, alias="n_parts")
34-
"""Number of parts to split the model into.
34+
"""Number of parts to split the model into.
3535
If -1, the number of parts is automatically determined."""
3636

3737
seed: int = Field(-1, alias="seed")
3838
"""Seed. If -1, a random seed is used."""
3939

40-
f16_kv: bool = Field(False, alias="f16_kv")
40+
f16_kv: bool = Field(True, alias="f16_kv")
4141
"""Use half-precision for key/value cache."""
4242

4343
logits_all: bool = Field(False, alias="logits_all")
@@ -50,7 +50,7 @@ class LlamaCpp(LLM):
5050
"""Force system to keep model in RAM."""
5151

5252
n_threads: Optional[int] = Field(None, alias="n_threads")
53-
"""Number of threads to use.
53+
"""Number of threads to use.
5454
If None, the number of threads is automatically determined."""
5555

5656
n_batch: Optional[int] = Field(8, alias="n_batch")

0 commit comments

Comments
 (0)