From 7613d23db7bc3cfce7d4123f220dcfa6657236ee Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 17 Jul 2024 18:06:58 -0400 Subject: [PATCH 001/155] feat: Update llama.cpp --- llama_cpp/llama.py | 23 ++++++--- llama_cpp/llama_cpp.py | 114 ++++++++++++++++++++++++++++------------- vendor/llama.cpp | 2 +- 3 files changed, 95 insertions(+), 44 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d8c2e0cdd..4b6f6b8f4 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -408,15 +408,24 @@ def __init__( ) ) + self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None + if self.lora_path: - if self._model.apply_lora_from_file( - self.lora_path, - self.lora_scale, - self.lora_base, - self.n_threads, + assert self._model.model is not None + self._lora_adapter = llama_cpp.llama_lora_adapter_init( + self._model.model, + self.lora_path.encode("utf-8"), + ) + if self._lora_adapter is None: + raise RuntimeError( + f"Failed to initialize LoRA adapter from lora path: {self.lora_path}" + ) + assert self._ctx.ctx is not None + if llama_cpp.llama_lora_adapter_set( + self._ctx.ctx, self._lora_adapter, self.lora_scale ): raise RuntimeError( - f"Failed to apply LoRA from lora path: {self.lora_path} to base path: {self.lora_base}" + f"Failed to set LoRA adapter from lora path: {self.lora_path}" ) if self.verbose: @@ -2077,6 +2086,8 @@ def close(self) -> None: self._stack.close() def __del__(self) -> None: + if self._lora_adapter is not None: + llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index decc08e65..35dc030b1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -401,7 +401,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 +# // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 # // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed # // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed # LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors @@ -430,14 +430,16 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors # LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors - +# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; LLAMA_FTYPE_ALL_F32 = 0 LLAMA_FTYPE_MOSTLY_F16 = 1 LLAMA_FTYPE_MOSTLY_Q4_0 = 2 LLAMA_FTYPE_MOSTLY_Q4_1 = 3 -LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 LLAMA_FTYPE_MOSTLY_Q8_0 = 7 LLAMA_FTYPE_MOSTLY_Q5_0 = 8 LLAMA_FTYPE_MOSTLY_Q5_1 = 9 @@ -464,6 +466,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 LLAMA_FTYPE_MOSTLY_IQ1_M = 31 LLAMA_FTYPE_MOSTLY_BF16 = 32 +LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 +LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 +LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { @@ -1100,6 +1105,12 @@ class llama_chat_message(ctypes.Structure): ] +# // lora adapter +# struct llama_lora_adapter; +llama_lora_adapter_p = ctypes.c_void_p +llama_lora_adapter_p_ctypes = ctypes.POINTER(ctypes.c_void_p) + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -1507,43 +1518,72 @@ def llama_model_quantize( ... -# // Apply a LoRA adapter to a loaded model -# // path_base_model is the path to a higher quality model to use as a base for -# // the layers modified by the adapter. Can be NULL to use the current loaded model. -# // The model needs to be reloaded before applying a new adapter, otherwise the adapter -# // will be applied on top of the previous one -# // Returns 0 on success -# LLAMA_API int32_t llama_model_apply_lora_from_file( -# const struct llama_model * model, -# const char * path_lora, -# float scale, -# const char * path_base_model, -# int32_t n_threads); -@ctypes_function( - "llama_model_apply_lora_from_file", - [ - llama_model_p_ctypes, - ctypes.c_char_p, - ctypes.c_float, - ctypes.c_char_p, - ctypes.c_int32, - ], +# // Load a LoRA adapter from file +# // The loaded adapter will be associated to the given model, and will be free when the model is deleted +# LLAMA_API struct llama_lora_adapter * llama_lora_adapter_init( +# struct llama_model * model, +# const char * path_lora); +@ctypes_function( + "llama_lora_adapter_init", + [llama_model_p_ctypes, ctypes.c_char_p], + llama_lora_adapter_p_ctypes, +) +def llama_lora_adapter_init( + model: llama_model_p, path_lora: bytes, / +) -> Optional[llama_lora_adapter_p]: + """Load a LoRA adapter from file + The loaded adapter will be associated to the given model, and will be free when the model is deleted""" + ... + + +# // Add a loaded LoRA adapter to given context +# // This will not modify model's weight +# LLAMA_API int32_t llama_lora_adapter_set( +# struct llama_context * ctx, +# struct llama_lora_adapter * adapter, +# float scale); +@ctypes_function( + "llama_lora_adapter_set", + [llama_context_p_ctypes, llama_lora_adapter_p_ctypes, ctypes.c_float], ctypes.c_int32, ) -def llama_model_apply_lora_from_file( - model: llama_model_p, - path_lora: Union[ctypes.c_char_p, bytes], - scale: Union[ctypes.c_float, float], - path_base_model: Union[ctypes.c_char_p, bytes, None], - n_threads: Union[ctypes.c_int32, int], - /, +def llama_lora_adapter_set( + ctx: llama_context_p, adapter: llama_lora_adapter_p, scale: float, / +) -> int: + """Add a loaded LoRA adapter to given context + This will not modify model's weight""" + ... + + +# // Remove a LoRA adapter from given context +# // Return -1 if the adapter is not present in the context +# LLAMA_API int32_t llama_lora_adapter_remove( +# struct llama_context * ctx, +# struct llama_lora_adapter * adapter); +@ctypes_function( + "llama_lora_adapter_remove", + [llama_context_p_ctypes, llama_lora_adapter_p_ctypes], + ctypes.c_int32, +) +def llama_lora_adapter_remove( + ctx: llama_context_p, adapter: llama_lora_adapter_p, / ) -> int: - """Apply a LoRA adapter to a loaded model - path_base_model is the path to a higher quality model to use as a base for - the layers modified by the adapter. Can be NULL to use the current loaded model. - The model needs to be reloaded before applying a new adapter, otherwise the adapter - will be applied on top of the previous one - Returns 0 on success""" + """Remove a LoRA adapter from given context + Return -1 if the adapter is not present in the context""" + ... + + +# // Manually free a LoRA adapter +# // Note: loaded adapters will be free when the associated model is deleted +# LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); +@ctypes_function( + "llama_lora_adapter_free", + [llama_lora_adapter_p_ctypes], + None, +) +def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): + """Manually free a LoRA adapter + Note: loaded adapters will be free when the associated model is deleted""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7fdb6f73e..30f80ca0b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 +Subproject commit 30f80ca0bcee58669ada7a94244eeccc8c4807cc From 66d5cddbc31976636cf560e5b0b86e4333e85f2b Mon Sep 17 00:00:00 2001 From: Grider Date: Wed, 17 Jul 2024 23:09:28 +0100 Subject: [PATCH 002/155] fix(server): Use split_mode from model settings (#1594) Co-authored-by: Andrei --- llama_cpp/server/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index ad39c1004..c486f8885 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -223,6 +223,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: **kwargs, # Model Params n_gpu_layers=settings.n_gpu_layers, + split_mode=settings.split_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, From 797f54c3171a8aa56b83459651ca8ecc7b3bdf26 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 17 Jul 2024 23:09:55 +0100 Subject: [PATCH 003/155] fix(docs): Update README.md typo (#1589) Missing ll --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5641ccaa8..b0dfdd5b5 100644 --- a/README.md +++ b/README.md @@ -755,7 +755,7 @@ pip install -e .[all] make clean ``` -You can also test out specific commits of `lama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require +You can also test out specific commits of `llama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require changes to the `llama_cpp/llama_cpp.py` file to match the new API (additional changes may be required elsewhere). ## FAQ From 070047662fe6261a32eaae0b380b52119d9af54e Mon Sep 17 00:00:00 2001 From: ddh0 Date: Wed, 17 Jul 2024 20:51:16 -0500 Subject: [PATCH 004/155] fix: Change repeat_penalty to 1.0 to match llama.cpp defaults (#1590) * reapply changes after sync with main branch * change repeat_penalty to 1.0 to match llama.cpp defaults --------- Co-authored-by: Andrei --- llama_cpp/_internals.py | 2 +- llama_cpp/llama.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index d5d3b2179..dcd4e17ff 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -721,7 +721,7 @@ class _LlamaSamplingParams: typical_p: float = 1.00 temp: float = 0.80 penalty_last_n: int = 64 - penalty_repeat: float = 1.10 + penalty_repeat: float = 1.0 penalty_freq: float = 0.00 penalty_present: float = 0.00 mirostat: int = 0 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4b6f6b8f4..005045f5c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -658,7 +658,7 @@ def sample( min_p: float = 0.05, typical_p: float = 1.0, temp: float = 0.80, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, frequency_penalty: float = 0.0, presence_penalty: float = 0.0, tfs_z: float = 1.0, @@ -733,7 +733,7 @@ def generate( min_p: float = 0.05, typical_p: float = 1.0, temp: float = 0.80, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, reset: bool = True, frequency_penalty: float = 0.0, presence_penalty: float = 0.0, @@ -751,7 +751,7 @@ def generate( Examples: >>> llama = Llama("models/ggml-7b.bin") >>> tokens = llama.tokenize(b"Hello, world!") - >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.1): + >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.0): ... print(llama.detokenize([token])) Args: @@ -1020,7 +1020,7 @@ def _create_completion( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1639,7 +1639,7 @@ def create_completion( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1736,7 +1736,7 @@ def __call__( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1833,7 +1833,7 @@ def create_chat_completion( max_tokens: Optional[int] = None, presence_penalty: float = 0.0, frequency_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, tfs_z: float = 1.0, mirostat_mode: int = 0, mirostat_tau: float = 5.0, From 3638f7333b8ece31c61f83b0cbbfc6c1dfe06814 Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Wed, 17 Jul 2024 18:52:04 -0700 Subject: [PATCH 005/155] feat: Add 'required' literal to ChatCompletionToolChoiceOption (#1597) --- llama_cpp/llama_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 573d7d651..bbb58afc3 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -275,7 +275,7 @@ class ChatCompletionNamedToolChoice(TypedDict): ChatCompletionToolChoiceOption = Union[ - Literal["none", "auto"], ChatCompletionNamedToolChoice + Literal["none", "auto", "required"], ChatCompletionNamedToolChoice ] From f95057a4750a14de28bcf05b7dd380475f9f9c20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 17:16:13 -0400 Subject: [PATCH 006/155] chore(deps): bump microsoft/setup-msbuild from 1.3 to 2 (#1585) Bumps [microsoft/setup-msbuild](https://github.com/microsoft/setup-msbuild) from 1.3 to 2. - [Release notes](https://github.com/microsoft/setup-msbuild/releases) - [Changelog](https://github.com/microsoft/setup-msbuild/blob/main/building-release.md) - [Commits](https://github.com/microsoft/setup-msbuild/compare/v1.3...v2) --- updated-dependencies: - dependency-name: microsoft/setup-msbuild dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3c0267095..0733a68c5 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -45,7 +45,7 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.3 + uses: microsoft/setup-msbuild@v2 with: vs-version: '[16.11,16.12)' From 5105f400f32d48b2eb4d8bac563072fb38912115 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 22 Jul 2024 15:20:12 -0400 Subject: [PATCH 007/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 ++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 35dc030b1..881b41b51 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -247,8 +247,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 6 -LLAMA_SESSION_VERSION = 6 +# define LLAMA_SESSION_VERSION 7 +LLAMA_SESSION_VERSION = 7 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ @@ -314,6 +314,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, # LLAMA_VOCAB_PRE_TYPE_VIKING = 18, # LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -335,6 +338,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 LLAMA_VOCAB_PRE_TYPE_VIKING = 18 LLAMA_VOCAB_PRE_TYPE_JAIS = 19 +LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 +LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 +LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 30f80ca0b..081fe431a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 30f80ca0bcee58669ada7a94244eeccc8c4807cc +Subproject commit 081fe431aa8fb6307145c4feb3eed4f48cab19f8 From 816d4912d9d2971198d2300a840ce4c100152502 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 22 Jul 2024 15:26:00 -0400 Subject: [PATCH 008/155] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47077d8af..d71e1609e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.83] + +- feat: Update llama.cpp to ggerganov/llama.cpp@081fe431aa8fb6307145c4feb3eed4f48cab19f8 +- feat: Add 'required' literal to ChatCompletionToolChoiceOption by @mjschock in #1597 +- fix: Change repeat_penalty to 1.0 to match llama.cpp defaults by @ddh0 in #1590 +- fix(docs): Update README.md typo by @ericcurtin in #1589 +- fix(server): Use split_mode from model settings by @grider-withourai in #1594 +- feat(ci): Dockerfile update base images and post-install cleanup by @Smartappli in #1530 + ## [0.2.82] - feat: Update llama.cpp to ggerganov/llama.cpp@7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 09ffb594e..65364d878 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.82" +__version__ = "0.2.83" From a14b49d64e760183504e1a5b37f4c9beb36a20ff Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 23 Jul 2024 21:26:20 -0400 Subject: [PATCH 009/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 234 ++++++++++++++++++++++++++--------------- vendor/llama.cpp | 2 +- 2 files changed, 153 insertions(+), 83 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 881b41b51..e70f2832e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1192,7 +1192,8 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): ... +def llama_numa_init(numa: int, /): + ... # // Call once at the end of the program - currently only used for MPI @@ -1217,7 +1218,8 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... +) -> Optional[llama_model_p]: + ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1226,7 +1228,8 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): ... +def llama_free_model(model: llama_model_p, /): + ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1239,7 +1242,8 @@ def llama_free_model(model: llama_model_p, /): ... ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... +) -> Optional[llama_context_p]: + ... # // Frees all allocated memory @@ -1260,87 +1264,104 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: ... +def llama_time_us() -> int: + ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: ... +def llama_max_devices() -> int: + ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: ... +def llama_supports_mmap() -> bool: + ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: ... +def llama_supports_mlock() -> bool: + ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: ... +def llama_supports_gpu_offload() -> bool: + ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: + ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: ... +def llama_n_batch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: ... +def llama_vocab_type(model: llama_model_p, /) -> int: + ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: ... +def llama_rope_type(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: ... +def llama_n_vocab(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... +def llama_n_embd(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... +def llama_n_layer(model: llama_model_p, /) -> int: + ... # // Get the model's RoPE frequency scaling factor @@ -1538,7 +1559,8 @@ def llama_lora_adapter_init( model: llama_model_p, path_lora: bytes, / ) -> Optional[llama_lora_adapter_p]: """Load a LoRA adapter from file - The loaded adapter will be associated to the given model, and will be free when the model is deleted""" + The loaded adapter will be associated to the given model, and will be free when the model is deleted + """ ... @@ -2092,7 +2114,8 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2120,7 +2143,8 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # LLAMA_API bool llama_state_save_file( @@ -2144,7 +2168,8 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2169,7 +2194,8 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2247,7 +2273,8 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2277,7 +2304,8 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # // @@ -2582,7 +2610,8 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: ... +) -> bytes: + ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2591,7 +2620,8 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: ... +) -> float: + ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2600,7 +2630,8 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: ... +) -> int: + ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2664,7 +2695,7 @@ def llama_token_nl(model: llama_model_p, /) -> int: # // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); +# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32) def llama_add_bos_token(model: llama_model_p, /) -> int: """Returns -1 if unknown, 1 for true or 0 for false.""" @@ -2672,7 +2703,7 @@ def llama_add_bos_token(model: llama_model_p, /) -> int: # // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); +# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32) def llama_add_eos_token(model: llama_model_p, /) -> int: """Returns -1 if unknown, 1 for true or 0 for false.""" @@ -2689,17 +2720,20 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: ... +def llama_token_middle(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: ... +def llama_token_suffix(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: ... +def llama_token_eot(model: llama_model_p, /) -> int: + ... # // @@ -2861,6 +2895,11 @@ def llama_detokenize( ... +# // +# // Chat templates +# // + + # /// Apply chat template. Inspired by hf apply_chat_template() on python. # /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" # /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template @@ -2895,7 +2934,8 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: ... +) -> int: + ... # // @@ -2950,6 +2990,79 @@ def llama_grammar_copy(grammar: llama_grammar_p, /) -> llama_grammar_p: ... +# /// @details Apply constraints from grammar +# LLAMA_API void llama_grammar_sample( +# const struct llama_grammar * grammar, +# const struct llama_context * ctx, +# llama_token_data_array * candidates); +@ctypes_function( + "llama_grammar_sample", + [ + llama_grammar_p, + llama_context_p_ctypes, + llama_token_data_array_p, + ], + None, +) +def llama_grammar_sample( + grammar: llama_grammar_p, + ctx: llama_context_p, + candidates: Union[ + CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ], + /, +): + """Apply constraints from grammar""" + ... + + +# LLAMA_API DEPRECATED(void llama_sample_grammar( +# struct llama_context * ctx, +# llama_token_data_array * candidates, +# const struct llama_grammar * grammar), +# "use llama_grammar_sample instead"); +@ctypes_function( + "llama_sample_grammar", + [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], + None, +) +def llama_sample_grammar( + ctx: llama_context_p, + candidates: Union[ + CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ], + grammar, # type: llama_grammar_p + /, +): + """Apply constraints from grammar + + Parameters: + candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + grammar: A grammar object containing the rules and constraints to apply to the generated text. + """ + ... + + +# /// @details Accepts the sampled token into the grammar +# LLAMA_API void llama_grammar_accept_token( +# struct llama_grammar * grammar, +# struct llama_context * ctx, +# llama_token token); +@ctypes_function( + "llama_grammar_accept_token", + [llama_grammar_p, llama_context_p_ctypes, llama_token], + None, +) +def llama_grammar_accept_token( + grammar: llama_grammar_p, + ctx: llama_context_p, + token: Union[llama_token, int], + /, +): + """Accepts the sampled token into the grammar""" + ... + + # // # // Sampling functions # // @@ -3236,33 +3349,6 @@ def llama_sample_temp( ... -# /// @details Apply constraints from grammar -# LLAMA_API void llama_sample_grammar( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const struct llama_grammar * grammar); -@ctypes_function( - "llama_sample_grammar", - [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], - None, -) -def llama_sample_grammar( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - grammar, # type: llama_grammar_p - /, -): - """Apply constraints from grammar - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - grammar: A grammar object containing the rules and constraints to apply to the generated text. - """ - ... - - # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3395,23 +3481,6 @@ def llama_sample_token( ... -# /// @details Accepts the sampled token into the grammar -# LLAMA_API void llama_grammar_accept_token( -# struct llama_context * ctx, -# struct llama_grammar * grammar, -# llama_token token); -@ctypes_function( - "llama_grammar_accept_token", - [llama_context_p_ctypes, llama_grammar_p, llama_token], - None, -) -def llama_grammar_accept_token( - ctx: llama_context_p, grammar: llama_grammar_p, token: Union[llama_token, int], / -) -> None: - """Accepts the sampled token into the grammar""" - ... - - # // # // Model split # // @@ -3533,4 +3602,5 @@ def llama_log_set( [ctypes.c_void_p, llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ... +def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 081fe431a..b841d0740 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 081fe431aa8fb6307145c4feb3eed4f48cab19f8 +Subproject commit b841d0740855c5af1344a81f261139a45a2b39ee From dccb148580e4794879cd66ff6c5bea6af95a66e2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 28 Jul 2024 11:58:22 -0400 Subject: [PATCH 010/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 91 ++++++++++++++++++++++++++++++------------ vendor/llama.cpp | 2 +- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index e70f2832e..d563d77bc 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -233,9 +233,6 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_DEFAULT_SEED 0xFFFFFFFF LLAMA_DEFAULT_SEED = 0xFFFFFFFF -# define LLAMA_MAX_RNG_STATE (64*1024) -LLAMA_MAX_RNG_STATE = 64 * 1024 - # define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' LLAMA_FILE_MAGIC_GGLA = 0x67676C61 @@ -247,13 +244,13 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 7 -LLAMA_SESSION_VERSION = 7 +# define LLAMA_SESSION_VERSION 8 +LLAMA_SESSION_VERSION = 8 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ -# define LLAMA_STATE_SEQ_VERSION 1 -LLAMA_STATE_SEQ_VERSION = 1 +# define LLAMA_STATE_SEQ_VERSION 2 +LLAMA_STATE_SEQ_VERSION = 2 # struct llama_model; llama_model_p = NewType("llama_model_p", int) @@ -1583,7 +1580,7 @@ def llama_lora_adapter_set( ... -# // Remove a LoRA adapter from given context +# // Remove a specific LoRA adapter from given context # // Return -1 if the adapter is not present in the context # LLAMA_API int32_t llama_lora_adapter_remove( # struct llama_context * ctx, @@ -1601,6 +1598,19 @@ def llama_lora_adapter_remove( ... +# // Remove all LoRA adapters from given context +# LLAMA_API void llama_lora_adapter_clear( +# struct llama_context * ctx); +@ctypes_function( + "llama_lora_adapter_clear", + [llama_context_p_ctypes], + None, +) +def llama_lora_adapter_clear(ctx: llama_context_p, /): + """Remove all LoRA adapters from given context""" + ... + + # // Manually free a LoRA adapter # // Note: loaded adapters will be free when the associated model is deleted # LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); @@ -1992,17 +2002,17 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // -# Returns the maximum size in bytes of the state (rng, logits, embedding -# and kv_cache) - will often be smaller after compacting tokens -# LLAMA_API size_t llama_state_get_size(const struct llama_context * ctx); +# // Returns the *actual* size in bytes of the state +# // (rng, logits, embedding and kv_cache) +# // Only use when saving the state, not when restoring it, otherwise the size may be too small. +# LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the maximum size in bytes of the state (rng, logits, embedding - and kv_cache) - will often be smaller after compacting tokens""" + """Returns the *actual* size in bytes of the state (rng, logits, embedding and kv_cache) - will often be smaller after compacting tokens""" ... -# LLAMA_API DEPRECATED(size_t llama_get_state_size(const struct llama_context * ctx), +# LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), # "use llama_state_get_size instead"); @ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_get_state_size(ctx: llama_context_p, /) -> int: @@ -2011,22 +2021,27 @@ def llama_get_state_size(ctx: llama_context_p, /) -> int: ... -# Copies the state to the specified destination address. -# Destination needs to have allocated enough memory. -# Returns the number of bytes copied +# // Copies the state to the specified destination address. +# // Destination needs to have allocated enough memory. +# // Returns the number of bytes copied # LLAMA_API size_t llama_state_get_data( # struct llama_context * ctx, -# uint8_t * dst); +# uint8_t * dst, +# size_t size); @ctypes_function( "llama_state_get_data", [ llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, ], ctypes.c_size_t, ) def llama_state_get_data( - ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / + ctx: llama_context_p, + dst: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + /, ) -> int: """Copies the state to the specified destination address. Destination needs to have allocated enough memory. @@ -2059,14 +2074,18 @@ def llama_copy_state_data( # // Returns the number of bytes read # LLAMA_API size_t llama_state_set_data( # struct llama_context * ctx, -# const uint8_t * src); +# const uint8_t * src, +# size_t size); @ctypes_function( "llama_state_set_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)], + [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t], ctypes.c_size_t, ) def llama_state_set_data( - ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / + ctx: llama_context_p, + src: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + /, ) -> int: """Set the state reading from the specified address Returns the number of bytes read""" @@ -2216,14 +2235,24 @@ def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> i # LLAMA_API size_t llama_state_seq_get_data( # struct llama_context * ctx, # uint8_t * dst, +# size_t size, # llama_seq_id seq_id); @ctypes_function( "llama_state_seq_get_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + ], ctypes.c_size_t, ) def llama_state_seq_get_data( - ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], seq_id: llama_seq_id, / + ctx: llama_context_p, + dst: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + seq_id: llama_seq_id, + /, ) -> int: """Copy the KV cache of a single sequence into the specified buffer""" ... @@ -2236,14 +2265,24 @@ def llama_state_seq_get_data( # LLAMA_API size_t llama_state_seq_set_data( # struct llama_context * ctx, # const uint8_t * src, +# size_t size, # llama_seq_id dest_seq_id); @ctypes_function( "llama_state_seq_set_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + ], ctypes.c_size_t, ) def llama_state_seq_set_data( - ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], dest_seq_id: llama_seq_id, / + ctx: llama_context_p, + src: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + dest_seq_id: llama_seq_id, + /, ) -> int: """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b841d0740..4730faca6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b841d0740855c5af1344a81f261139a45a2b39ee +Subproject commit 4730faca618ff9cee0780580145e3cbe86f24876 From 9ed6b2727c0790a106d4be8c4d25b1d6eceff050 Mon Sep 17 00:00:00 2001 From: Abhay Nigam <36666689+mashuk999@users.noreply.github.com> Date: Sun, 28 Jul 2024 21:29:01 +0530 Subject: [PATCH 011/155] fix: Correcting run.sh filepath in Simple Docker implementation (#1626) --- docker/simple/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 2838fd1ff..3594df1a5 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -35,4 +35,4 @@ ENV PORT=8000 EXPOSE 8000 # Run the server start script -CMD ["/bin/sh", "/app/run.sh"] +CMD ["/bin/sh", "/app/docker/simple/run.sh"] From 4bf3b43df95e62ddf74413a22a1df3411d206acc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 28 Jul 2024 12:06:36 -0400 Subject: [PATCH 012/155] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d71e1609e..4d74c3032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.84] + +- feat: Update llama.cpp to ggerganov/llama.cpp@4730faca618ff9cee0780580145e3cbe86f24876 +- fix: fix: Correcting run.sh filepath in Simple Docker implementation by @mashuk999 in #1626 + ## [0.2.83] - feat: Update llama.cpp to ggerganov/llama.cpp@081fe431aa8fb6307145c4feb3eed4f48cab19f8 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 65364d878..d94a23db5 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.83" +__version__ = "0.2.84" From cffb4ec8c40e556e10b4e1168a43ba7f18bd957d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Jul 2024 22:11:49 -0400 Subject: [PATCH 013/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4730faca6..268c56600 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4730faca618ff9cee0780580145e3cbe86f24876 +Subproject commit 268c5660062270a2c19a36fc655168aa287aaec2 From 53c6f329595bc3d8d242f58e0f1341d57bedfd92 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 31 Jul 2024 12:10:19 -0400 Subject: [PATCH 014/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 268c56600..398ede5ef 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 268c5660062270a2c19a36fc655168aa287aaec2 +Subproject commit 398ede5efeb07b9adf9fbda7ea63f630d476a792 From 0b1a8d8eff24595554513f0586929cba3e583bc2 Mon Sep 17 00:00:00 2001 From: "yuri@FreeBSD" Date: Wed, 31 Jul 2024 09:12:19 -0700 Subject: [PATCH 015/155] feat: FreeBSD compatibility (#1635) --- llama_cpp/llama_cpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d563d77bc..727195cf5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -28,7 +28,7 @@ def _load_shared_library(lib_base_name: str): # for llamacpp) and "llama" (default name for this repo) _lib_paths: List[pathlib.Path] = [] # Determine the file extension based on the platform - if sys.platform.startswith("linux"): + if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): _lib_paths += [ _base_path / f"lib{lib_base_name}.so", ] From 8297a0d32ea1b5d88451a1805323a0a4bde466b2 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:00 +0200 Subject: [PATCH 016/155] fix(docker): Update Dockerfile build options from `LLAMA_` to `GGML_` (#1634) --- docker/openblas_simple/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/openblas_simple/Dockerfile b/docker/openblas_simple/Dockerfile index 804c1b902..5ee667dc0 100644 --- a/docker/openblas_simple/Dockerfile +++ b/docker/openblas_simple/Dockerfile @@ -12,7 +12,7 @@ RUN apt update && apt install -y libopenblas-dev ninja-build build-essential pkg RUN python -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context -RUN CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama_cpp_python --verbose +RUN CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama_cpp_python --verbose # Run the server CMD python3 -m llama_cpp.server From ac0217404c72a66bd47a84fa741a09cf712b8975 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:24 +0200 Subject: [PATCH 017/155] fix(docker): Fix GGML_CUDA param (#1633) --- docker/cuda_simple/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cuda_simple/Dockerfile b/docker/cuda_simple/Dockerfile index 79b6a5bac..0bbf20ffe 100644 --- a/docker/cuda_simple/Dockerfile +++ b/docker/cuda_simple/Dockerfile @@ -15,13 +15,13 @@ COPY . . # setting build related env vars ENV CUDA_DOCKER_ARCH=all -ENV LLAMA_CUBLAS=1 +ENV GGML_CUDA=1 # Install depencencies RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context # Install llama-cpp-python (build with cuda) -RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +RUN CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python # Run the server CMD python3 -m llama_cpp.server From 8a12c9f34680c4bd3e6c84557f386a7253b8fbbe Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:53 +0200 Subject: [PATCH 018/155] fix(docker): Update Dockerfile BLAS options (#1632) --- docker/open_llama/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/open_llama/Dockerfile b/docker/open_llama/Dockerfile index 70b16dffc..f05e66ef2 100644 --- a/docker/open_llama/Dockerfile +++ b/docker/open_llama/Dockerfile @@ -20,13 +20,13 @@ RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fa # Perform the conditional installations based on the image RUN echo "Image: ${IMAGE}" && \ - if [ "${IMAGE}" = "python:3-slim-bullseye" ] ; then \ + if [ "${IMAGE}" = "python:3-slim-bookworm" ] ; then \ echo "OpenBLAS install:" && \ apt-get install -y --no-install-recommends libopenblas-dev && \ - LLAMA_OPENBLAS=1 pip install llama-cpp-python --verbose; \ + CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python --verbose; \ else \ echo "CuBLAS install:" && \ - LLAMA_CUBLAS=1 pip install llama-cpp-python --verbose; \ + CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python --verbose; \ fi # Clean up apt cache From 1f0b9a2872065a89c16a4567b561ddd10f2533fe Mon Sep 17 00:00:00 2001 From: Shamit Verma Date: Wed, 31 Jul 2024 21:45:51 +0530 Subject: [PATCH 019/155] fix : Missing LoRA adapter after API change (#1630) --- llama_cpp/llama.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 005045f5c..0cb5ca2fc 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2083,11 +2083,14 @@ def pooling_type(self) -> str: def close(self) -> None: """Explicitly free the model from memory.""" - self._stack.close() + if hasattr(self,'_stack'): + if self._stack is not None: + self._stack.close() def __del__(self) -> None: - if self._lora_adapter is not None: - llama_cpp.llama_lora_adapter_free(self._lora_adapter) + if hasattr(self,'_lora_adapter'): + if self._lora_adapter is not None: + llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod From f7b9e6d42981bb75106699e0712a6378c39ef92c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 31 Jul 2024 12:24:21 -0400 Subject: [PATCH 020/155] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d74c3032..6dc25442b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.85] + +- feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 +- fix: Missing LoRA adapter after API change by @shamitv in #1630 +- fix(docker): Update Dockerfile BLAS options by @olivierdebauche in #1632 +- fix(docker): Fix GGML_CUDA param by @olivierdebauche in #1633 +- fix(docker): Update Dockerfile build options from `LLAMA_` to `GGML_` by @olivierdebauche in #1634 +- feat: FreeBSD compatibility by @yurivict in #1635 + ## [0.2.84] - feat: Update llama.cpp to ggerganov/llama.cpp@4730faca618ff9cee0780580145e3cbe86f24876 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d94a23db5..97c8d8174 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.84" +__version__ = "0.2.85" From 5575fedd33141ffdeb87304da3b1225da58db76a Mon Sep 17 00:00:00 2001 From: tc-wolf <50339167+tc-wolf@users.noreply.github.com> Date: Sun, 4 Aug 2024 16:07:54 -0500 Subject: [PATCH 021/155] fix: llama_grammar_accept_token arg order (#1649) Old was: llama_grammar_accept_token(ctx, grammar, token) Now this is: llama_grammar_accept_token(grammar, ctx, token) --- llama_cpp/_internals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index dcd4e17ff..357023e3c 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -511,7 +511,7 @@ def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: def grammar_accept_token(self, grammar: LlamaGrammar, token: int): assert self.ctx is not None assert grammar.grammar is not None - llama_cpp.llama_grammar_accept_token(self.ctx, grammar.grammar, token) + llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): assert self.ctx is not None From dff186cc5911a03c1c2de2505ab467c419aa6821 Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Wed, 7 Aug 2024 02:19:36 +0200 Subject: [PATCH 022/155] feat: Ported back new grammar changes from C++ to Python implementation (#1637) * Backported . (any chat) from llama.cpp * unfinished {count,optionalmax) * implemented slice function in std:vector * fixed mistake done while reading * ported https://github.com/ggerganov/llama.cpp/pull/7194 * multiple fixes, var copy * Rewrite LlamaGrammar internals in python style * bugfix --------- Co-authored-by: Andrei --- llama_cpp/llama_cpp.py | 2 +- llama_cpp/llama_grammar.py | 1652 +++++++++++++++--------------------- 2 files changed, 682 insertions(+), 972 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 727195cf5..d598cf9f1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3002,7 +3002,7 @@ def llama_grammar_init( n_rules: Union[ctypes.c_size_t, int], start_rule_index: Union[ctypes.c_size_t, int], /, -) -> llama_grammar_p: +) -> Optional[llama_grammar_p]: """Initialize a grammar from a set of rules.""" ... diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 0ac7354bb..e60817e1d 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1,520 +1,101 @@ """Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.""" # flake8: noqa -from pathlib import Path import sys -from ctypes import * # type: ignore -from enum import Enum -from itertools import islice, groupby +import ctypes +import enum +import typing +import dataclasses + +from itertools import groupby from typing import ( Any, - Callable, - Dict, Set, - Generic, List, Optional, - OrderedDict, - TextIO, Tuple, - TypeVar, Union, - overload, ) import llama_cpp.llama_cpp as llama_cpp -# Type aliases -llama_grammar_element = llama_cpp.llama_grammar_element -llama_grammar_element_p = llama_cpp.llama_grammar_element_p -llama_grammar_p = llama_cpp.llama_grammar_p - -# Type variables -Ptr = TypeVar("Ptr", bound="const_char_p") -T = TypeVar("T") -U = TypeVar("U") -V = TypeVar("V") -W = TypeVar("W") - - -class Sentinel: - """Used to mark the end of a iterator of std::vector & std::map.""" - - -class LlamaGrammar: - """Keeps reference counts of all the arguments, so that they are not - garbage collected by Python.""" - - def __del__(self) -> None: - """Free the grammar pointer when the object is deleted.""" - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.grammar = None - - def __init__( - self, - parsed_grammar: "parse_state", - ) -> None: - """Initialize the grammar pointer from the parsed state.""" - self._grammar_rules = ( - parsed_grammar.c_rules() - ) # type: std.vector[std.vector[LlamaGrammarElement]] - self._n_rules = self._grammar_rules.size() # type: int - self._start_rule_index = parsed_grammar.symbol_ids.at("root") # type: int - self.init() - - @classmethod - def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": - """Convert a GBNF grammar to a Llama grammar.""" - parsed_grammar = parse(const_char_p(grammar)) # type: parse_state - if parsed_grammar.rules.empty(): - raise ValueError( - f"{cls.from_string.__name__}: error parsing grammar file: parsed_grammar.rules is empty" - ) - if verbose: - print(f"{cls.from_string.__name__} grammar:", file=sys.stderr) - print_grammar(sys.stderr, parsed_grammar) - print(file=sys.stderr) - return cls(parsed_grammar) - - @classmethod - def from_json_schema( - cls, - json_schema: str, - verbose: bool = True, - ) -> "LlamaGrammar": - """Convert a JSON schema to a Llama grammar.""" - return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose) - - @classmethod - def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": - try: - with open(file) as f: - grammar = f.read() - except Exception as err: - raise Exception( - f"{cls.from_file.__name__}: error reading grammar file: {err}" - ) - - if grammar: - return cls.from_string(grammar, verbose=verbose) - - raise ValueError( - f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty" - ) - - def init(self) -> None: - # Step 1: Convert LlamaGrammarElement to llama_grammar_element - self._element_lists = [ - [ - llama_grammar_element(c_int(elem.type.value), c_uint32(elem.value)) - for elem in subvector - ] - for subvector in self._grammar_rules - ] # type: List[List[llama_grammar_element]] - - # Step 2: Convert each list to llama_grammar_element array and get pointer - self._element_arrays = [ - (llama_grammar_element * len(sublist))(*sublist) - for sublist in self._element_lists - ] # type: List[Array[llama_grammar_element]] - - # Step 3: Get pointer of each array - self._element_array_pointers = [ - cast(subarray, llama_grammar_element_p) for subarray in self._element_arrays - ] # type: List[llama_grammar_element_p] - - # Step 4: Make array of these pointers and get its pointer - self._rules = (llama_grammar_element_p * len(self._element_array_pointers))( - *self._element_array_pointers - ) - self.grammar = llama_cpp.llama_grammar_init( - self._rules, c_size_t(self._n_rules), c_size_t(self._start_rule_index) - ) +class GrammarElementType(enum.IntEnum): + END = llama_cpp.LLAMA_GRETYPE_END + ALT = llama_cpp.LLAMA_GRETYPE_ALT + RULE_REF = llama_cpp.LLAMA_GRETYPE_RULE_REF + CHAR = llama_cpp.LLAMA_GRETYPE_CHAR + CHAR_NOT = llama_cpp.LLAMA_GRETYPE_CHAR_NOT + CHAR_RNG_UPPER = llama_cpp.LLAMA_GRETYPE_CHAR_RNG_UPPER + CHAR_ALT = llama_cpp.LLAMA_GRETYPE_CHAR_ALT + CHAR_ANY = llama_cpp.LLAMA_GRETYPE_CHAR_ANY - def reset(self) -> None: - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.init() +@dataclasses.dataclass +class GrammarElement: + type: GrammarElementType + value: int -class LlamaGrammarElement: - def __init__(self, type: "llama_gretype", value: int): - self.type = type - self.value = value # Unicode code point or rule ID +@dataclasses.dataclass +class ParseState: + symbol_ids: typing.Dict[str, int] = dataclasses.field(default_factory=dict) + rules: typing.List[typing.List[GrammarElement]] = dataclasses.field(default_factory=list) -class const_char_p: - """C++ implementation of const char *.""" - - def __init__(self, value: Union[str, Ptr], move: Optional[int] = None): - if isinstance(value, const_char_p): - # We're copying an existing const_char_p - self.value = value.value - self.pos = value.pos + (move or 0) - return - - # We're creating a new const_char_p - self.value = value - self.pos = move or 0 - - def __str__(self) -> str: - assert self.value is not None, "null pointer" - return self.value[self.pos :] - - def __getitem__(self, index: int) -> str: - value = str(self) - return value[index] if index < len(value) else "" - - @overload - def __add__(self: Ptr, other: int) -> Ptr: ... - - @overload - def __add__(self: Ptr, other: Ptr) -> int: ... - - def __add__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: - return ( - self.__class__(self.value, self.pos + other) - if isinstance(other, int) - else self.pos + other.pos - ) - - @overload - def __sub__(self: Ptr, other: int) -> Ptr: ... - - @overload - def __sub__(self: Ptr, other: Ptr) -> int: ... - - def __sub__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: - return ( - self.__class__(self.value, self.pos - other) - if isinstance(other, int) - else self.pos - other.pos - ) - - def __eq__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos == other.pos - - def __lt__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos < other.pos - - def __gt__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos > other.pos - - -class std: - @staticmethod - def string(ptr: const_char_p, length: Optional[int] = None) -> str: - """C++ implementation of std::string constructor.""" - value = str(ptr) - if length is not None: - value = value[:length] - return value - - class vector(Generic[T], List[T]): - """C++ implementation of std::vector.""" - - class iterator: - def __init__(self, vector: "std.vector[T]", index: int): - self._vector = vector - self._index = index - self._version = vector._version - - def _check_version(self): - if self._version != self._vector._version: - raise RuntimeError("Iterator used after vector was modified.") - - def __iter__(self): - return self - - def __next__(self) -> T: - self._check_version() - if self._index >= self._vector.size(): - raise StopIteration - value = self._vector[self._index] - self._index += 1 - return value - - def __add__(self, value: int) -> "std.vector[T].iterator": - return self.__class__(self._vector, self._index + value) - - def __sub__(self, value: int) -> "std.vector[T].iterator": - return self.__class__(self._vector, self._index - value) - - def __init__(self): - self._version = 0 - - def modify(self): - # This is a bit of a hack to make sure iterators are invalidated - self._version += 1 - - def push_back(self, value: T) -> None: - self.modify() - self.append(value) - - def pop_back(self) -> None: - self.modify() - if not self.empty(): - self.pop() - - def back(self) -> T: - return self[-1] - - def size(self) -> int: - return len(self) - - def clear(self) -> None: - self.modify() - super().clear() - - def empty(self) -> bool: - return self.size() == 0 - - def data(self) -> "std.vector[T]": - return self - - def resize( - self, - new_size: int, - fill_value_factory: Optional[Callable[[], T]] = None, - ) -> None: - if new_size > self.size(): - if fill_value_factory is None: - raise ValueError("A fill value factory function must be provided.") - self.reserve(new_size, fill_value_factory) - elif new_size < self.size(): - self[:] = self[:new_size] - - def reserve(self, capacity: int, fill_value_factory: Callable[[], T]) -> None: - if capacity > self.size(): - fill_value = fill_value_factory() - self.extend([fill_value] * (capacity - self.size())) - - def front(self) -> T: - if not self.empty(): - return self[0] - else: - raise IndexError("Vector is empty.") - - def assign(self, count: int, value: T) -> None: - self.clear() - self.extend([value] * count) - - def insert( - self, - pos: "std.vector[T].iterator", - first: "std.vector[T].iterator", - last: "std.vector[T].iterator", - ) -> None: - self[pos._index : pos._index] = list( - islice(first._vector, first._index, last._index) - ) - - def begin(self) -> "std.vector[T].iterator": - return self.iterator(self, 0) - - def end(self) -> "std.vector[T].iterator": - return self.iterator(self, self.size()) - - class map(Generic[T, U], OrderedDict[T, U]): - """C++ implementation of std::map.""" - - class iterator(Generic[V, W]): - def __init__(self, _map: "std.map[T, U]", key: Union[T, Sentinel]): - self._map = _map - self.iter = iter(_map) - self.key = key - self._advance() - - def _sanitize_key(self) -> T: - if isinstance(self.key, Sentinel): - raise StopIteration - return self.key - - def _advance(self) -> None: - try: - while next(self.iter) != self.key: - pass - except StopIteration: - self.key = Sentinel() - - def __next__(self) -> Tuple[T, U]: - key = self._sanitize_key() - if key in self._map: - value = self._map[key] - self._advance() - return key, value - else: - raise StopIteration - - def get(self) -> Tuple[T, U]: - key = self._sanitize_key() - return key, self._map[key] - - @property - def first(self) -> T: - return self._sanitize_key() - - @property - def second(self) -> U: - return self._map[self._sanitize_key()] - - def insert( - self, key: T, value: U - ) -> Tuple["std.map[T, U].iterator[T, U]", bool]: - if key in self: - return self.iterator(self, key), False - else: - self[key] = value - return self.iterator(self, key), True - - def find(self, key: T) -> "std.map[T, U].iterator[T, U]": - if key in self: - return self.iterator(self, key) - else: - return self.end() - - def at(self, key: T) -> U: - if key in self: - return self[key] - else: - raise KeyError("The provided key is not found in the map.") - - def erase(self, iterator: "std.map[T, U].iterator[T, U]") -> None: - key = iterator.first - if key in self: - del self[key] - - def size(self) -> int: - return len(self) - - def empty(self) -> bool: - return self.size() == 0 - - def lower_bound(self, key: T) -> "std.map[T, U].iterator[T, U]": - try: - keys = sorted(list(self.keys())) # type: ignore - for k in keys: - if k >= key: - return self.iterator(self, k) - raise ValueError("No key found that is not less than the input key") - except TypeError: - raise TypeError("Keys of type T cannot be sorted.") - - def begin(self) -> "std.map[T, U].iterator[T, U]": - return self.iterator(self, next(iter(self))) - - def end(self) -> "std.map[T, U].iterator[T, U]": - return self.iterator(self, Sentinel()) - - -# // grammar element type -# enum llama_gretype { -# // end of rule definition -# LLAMA_GRETYPE_END = 0, - -# // start of alternate definition for rule -# LLAMA_GRETYPE_ALT = 1, - -# // non-terminal element: reference to rule -# LLAMA_GRETYPE_RULE_REF = 2, - -# // terminal element: character (code point) -# LLAMA_GRETYPE_CHAR = 3, - -# // inverse char(s) ([^a], [^a-b] [^abc]) -# LLAMA_GRETYPE_CHAR_NOT = 4, - -# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to -# // be an inclusive range ([a-z]) -# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, - - -# // modifies a preceding LLAMA_GRETYPE_CHAR or -# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) -# LLAMA_GRETYPE_CHAR_ALT = 6, -# }; -class llama_gretype(Enum): - """grammar element type""" - - LLAMA_GRETYPE_END = 0 # end of rule definition - LLAMA_GRETYPE_ALT = 1 # start of alternate definition for rule - LLAMA_GRETYPE_RULE_REF = 2 # non-terminal element: reference to rule - LLAMA_GRETYPE_CHAR = 3 # terminal element: character (code point) - LLAMA_GRETYPE_CHAR_NOT = 4 # inverse char(s) ([^a], [^a-b] [^abc]) - LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 # modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to be an inclusive range ([a-z]) - LLAMA_GRETYPE_CHAR_ALT = 6 # modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) - - -# struct parse_state { -# std::map symbol_ids; -# std::vector> rules; -# std::vector c_rules(); -# }; -class parse_state: - def __init__(self): - self.symbol_ids: std.map[str, int] = std.map() - self.rules: std.vector[std.vector[LlamaGrammarElement]] = std.vector() - - # std::vector parse_state::c_rules() { - # std::vector ret; - # for (const auto & rule : rules) { - # ret.push_back(rule.data()); - # } - # return ret; - # } - def c_rules(self) -> std.vector[std.vector[LlamaGrammarElement]]: - ret = std.vector() # type: std.vector[std.vector[LlamaGrammarElement]] - for rule in self.rules: - ret.push_back(rule.data()) - return ret - - def __repr__(self) -> str: - return ( - f"parse_state(symbol_ids={len(self.symbol_ids)}, rules={len(self.rules)})" - ) +# static std::pair decode_utf8(const char * src) { +# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; +# uint8_t first_byte = static_cast(*src); +# uint8_t highbits = first_byte >> 4; +# int len = lookup[highbits]; +# uint8_t mask = (1 << (8 - len)) - 1; +# uint32_t value = first_byte & mask; +# const char * end = src + len; // may overrun! +# const char * pos = src + 1; +# for ( ; pos < end && *pos; pos++) { +# value = (value << 6) + (static_cast(*pos) & 0x3F); +# } +# return std::make_pair(value, pos); +# } +def decode_utf8(src: str) -> typing.Tuple[int, str]: + lookup: list[int] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4] + first_byte: int = ord(src[0]) + highbits: int = first_byte >> 4 + length: int = lookup[highbits] + mask: int = (1 << (8 - length)) - 1 + value: int = first_byte & mask + end: int = min(len(src), length) # Prevent overrun + + pos: int = 1 + for pos in range(1, end): + if not src[pos]: + break + value = (value << 6) + (ord(src[pos]) & 0x3F) -# struct llama_grammar { -# const std::vector> rules; -# std::vector> stacks; -# }; -# class llama_grammar: -# def __init__( -# self, -# rules: std.vector[std.vector[llama_grammar_element]], -# stacks: std.vector[std.vector[llama_grammar_element]], -# ): -# self.rules = rules -# self.stacks = stacks + return value, src[pos:] if pos < len(src) else "" -# uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { +# static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { # uint32_t next_id = static_cast(state.symbol_ids.size()); -# auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id)); +# auto result = state.symbol_ids.emplace(std::string(src, len), next_id); # return result.first->second; # } -def get_symbol_id(state: parse_state, src: const_char_p, len: int) -> int: - next_id = state.symbol_ids.size() # type: int - result = state.symbol_ids.insert(std.string(src, len), next_id) - return result[0].second # type: ignore +def get_symbol_id(state: ParseState, name: str) -> int: + next_id = len(state.symbol_ids) + return state.symbol_ids.setdefault(name, next_id) -# uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { +# static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { # uint32_t next_id = static_cast(state.symbol_ids.size()); # state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id; # return next_id; # } -def generate_symbol_id(state: parse_state, base_name: str) -> int: - next_id = state.symbol_ids.size() # type: int - state.symbol_ids[base_name + "_" + str(next_id)] = next_id +def generate_symbol_id(state: ParseState, base_name: str) -> int: + next_id = len(state.symbol_ids) + state.symbol_ids[f"{base_name}_{next_id}"] = next_id return next_id -# void add_rule( +# static void add_rule( # parse_state & state, # uint32_t rule_id, # const std::vector & rule) { @@ -523,51 +104,27 @@ def generate_symbol_id(state: parse_state, base_name: str) -> int: # } # state.rules[rule_id] = rule; # } -def add_rule( - state: parse_state, - rule_id: int, - rule: std.vector[LlamaGrammarElement], -) -> None: - if state.rules.size() <= rule_id: - state.rules.resize( - rule_id + 1, - fill_value_factory=std.vector[LlamaGrammarElement], - ) +def add_rule(state: ParseState, rule_id: int, rule: typing.List[GrammarElement]) -> None: + if len(state.rules) <= rule_id: + state.rules.extend([[]] * (rule_id + 1 - len(state.rules))) state.rules[rule_id] = rule -# std::pair decode_utf8(const char * src) { -# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; -# uint8_t first_byte = static_cast(*src); -# uint8_t highbits = first_byte >> 4; -# int len = lookup[highbits]; -# uint8_t mask = (1 << (8 - len)) - 1; -# uint32_t value = first_byte & mask; -# const char * end = src + len; // may overrun! -# const char * pos = src + 1; -# for ( ; pos < end && *pos; pos++) { -# value = (value << 6) + (static_cast(*pos) & 0x3F); -# } -# return std::make_pair(value, pos); +# static bool is_digit_char(char c) { +# return '0' <= c && c <= '9'; # } -def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]: - """Decodes a UTF-8 character from the source string.""" - # Get the codepoint of the first character - value = ord(src[0]) - # Move the pointer ahead one character - pos = src + 1 - - return value, pos +def is_digit_char(c: str) -> bool: + return "0" <= c <= "9" -# bool is_word_char(char c) { -# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || ('0' <= c && c <= '9'); +# static bool is_word_char(char c) { +# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c); # } def is_word_char(c: str) -> bool: - return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or ("0" <= c <= "9") + return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or is_digit_char(c) -# std::pair parse_hex(const char * src, int size) { +# static std::pair parse_hex(const char * src, int size) { # const char * pos = src; # const char * end = src + size; # uint32_t value = 0; @@ -589,13 +146,12 @@ def is_word_char(c: str) -> bool: # } # return std::make_pair(value, pos); # } -def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: - pos = const_char_p(src) # type: const_char_p - end = src + size # type: const_char_p - value = 0 # type: int - while pos < end and pos[0]: +def parse_hex(src: str, size: int) -> typing.Tuple[int, str]: + pos = 0 + value = 0 + for _ in range(size): value <<= 4 - c = pos[0] # type: str + c = src[pos] if "a" <= c <= "f": value += ord(c) - ord("a") + 10 elif "A" <= c <= "F": @@ -605,12 +161,74 @@ def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: else: break pos += 1 - if pos != end: - raise RuntimeError("expecting " + str(size) + " hex chars at " + str(src)) - return (value, pos) + if pos != size: + raise ValueError(f"expecting {size} hex chars at {src}") + return value, src[pos:] + + +# static const char * parse_space(const char * src, bool newline_ok) { +# const char * pos = src; +# while (*pos == ' ' || *pos == '\t' || *pos == '#' || +# (newline_ok && (*pos == '\r' || *pos == '\n'))) { +# if (*pos == '#') { +# while (*pos && *pos != '\r' && *pos != '\n') { +# pos++; +# } +# } else { +# pos++; +# } +# } +# return pos; +# } +def parse_space(src: str, newline_ok: bool) -> str: + pos = src + while pos and (pos[0] in (' ', '\t', '#') or (newline_ok and pos[0] in ('\r', '\n'))): + if pos[0] == "#": + while pos and pos[0] not in ("\r", "\n"): + pos = pos[1:] + else: + pos = pos[1:] + return pos -# std::pair parse_char(const char * src) { +# static const char * parse_name(const char * src) { +# const char * pos = src; +# while (is_word_char(*pos)) { +# pos++; +# } +# if (pos == src) { +# throw std::runtime_error(std::string("expecting name at ") + src); +# } +# return pos; +# } +def parse_name(src: str) -> typing.Tuple[str, str]: + pos = src + while pos and is_word_char(pos[0]): + pos = pos[1:] + if pos == src: + raise ValueError(f"expecting name at {src}") + return src[:len(src) - len(pos)], pos + +# static const char * parse_int(const char * src) { +# const char * pos = src; +# while (is_digit_char(*pos)) { +# pos++; +# } +# if (pos == src) { +# throw std::runtime_error(std::string("expecting integer at ") + src); +# } +# return pos; +# } +def parse_int(src: str) -> typing.Tuple[int, str]: + pos = src + while pos and is_digit_char(pos[0]): + pos = pos[1:] + if pos == src: + raise ValueError(f"expecting integer at {src}") + return int(src[:len(src) - len(pos)]), pos + + +# static std::pair parse_char(const char * src) { # if (*src == '\\') { # switch (src[1]) { # case 'x': return parse_hex(src + 2, 2); @@ -632,273 +250,320 @@ def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: # } # throw std::runtime_error("unexpected end of input"); # } -def parse_char(src: const_char_p) -> Tuple[int, const_char_p]: +def parse_char(src: str) -> typing.Tuple[int, str]: + if not src: + raise ValueError("unexpected end of input") if src[0] == "\\": - case = src[1] # type: str - if case == "x": - return parse_hex(src + 2, 2) - elif case == "u": - return parse_hex(src + 2, 4) - elif case == "U": - return parse_hex(src + 2, 8) - elif case == "t": - return (ord("\t"), src + 2) # implicit cast - elif case == "r": - return (ord("\r"), src + 2) # implicit cast - elif case == "n": - return (ord("\n"), src + 2) # implicit cast - elif case in ("\\", '"', "[", "]"): - return (ord(case), src + 2) # implicit cast + if src[1] == "x": + return parse_hex(src[2:], 2) + elif src[1] == "u": + return parse_hex(src[2:], 4) + elif src[1] == "U": + return parse_hex(src[2:], 8) + elif src[1] == "t": + return ord("\t"), src[2:] + elif src[1] == "r": + return ord("\r"), src[2:] + elif src[1] == "n": + return ord("\n"), src[2:] + elif src[1] in ('\\', '"', '[', ']'): + return ord(src[1]), src[2:] else: - raise RuntimeError("unknown escape at " + str(src)) - elif src[0]: - return decode_utf8(src) - else: - raise RuntimeError("unexpected end of input") - - -# const char * parse_name(const char * src) { -# const char * pos = src; -# while (is_word_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting name at ") + src); -# } -# return pos; -# } -def parse_name(src: const_char_p) -> const_char_p: - pos = const_char_p(src) # type: const_char_p - while is_word_char(pos[0]): - pos += 1 - if pos == src: - raise RuntimeError("expecting name at " + str(src)) - return pos + raise ValueError(f"unknown escape at {src}") + return decode_utf8(src) - -# const char * parse_space(const char * src, bool newline_ok) { +# static const char * parse_sequence( +# parse_state & state, +# const char * src, +# const std::string & rule_name, +# std::vector & out_elements, +# bool is_nested) { +# size_t last_sym_start = out_elements.size(); # const char * pos = src; -# while (*pos == ' ' || *pos == '\t' || *pos == '#' || -# (newline_ok && (*pos == '\r' || *pos == '\n'))) { -# if (*pos == '#') { -# while (*pos && *pos != '\r' && *pos != '\n') { +# +# auto handle_repetitions = [&](int min_times, int max_times) { +# +# if (last_sym_start == out_elements.size()) { +# throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos); +# } +# +# // apply transformation to previous symbol (last_sym_start to end) according to +# // the following rewrite rules: +# // S{m,n} --> S S S (m times) S'(n-m) +# // S'(x) ::= S S'(x-1) | +# // (... n-m definitions of these S' rules ...) +# // S'(1) ::= S | +# // S{m,} --> S S S (m times) S' +# // S' ::= S S' | +# // S* --> S{0,} +# // --> S' ::= S S' | +# // S+ --> S{1,} +# // --> S S' +# // S' ::= S S' | +# // S? --> S{0,1} +# // --> S' +# // S' ::= S | +# +# std::vector previous_elements(out_elements.begin() + last_sym_start, out_elements.end()); +# if (min_times == 0) { +# out_elements.resize(last_sym_start); +# } else { +# // Repeat the previous elements (min_times - 1) times +# for (int i = 1; i < min_times; i++) { +# out_elements.insert(out_elements.end(), previous_elements.begin(), previous_elements.end()); +# } +# } +# +# uint32_t last_rec_rule_id = 0; +# auto n_opt = max_times < 0 ? 1 : max_times - min_times; +# +# std::vector rec_rule(previous_elements); +# for (int i = 0; i < n_opt; i++) { +# rec_rule.resize(previous_elements.size()); +# uint32_t rec_rule_id = generate_symbol_id(state, rule_name); +# if (i > 0 || max_times < 0) { +# rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, max_times < 0 ? rec_rule_id : last_rec_rule_id}); +# } +# rec_rule.push_back({LLAMA_GRETYPE_ALT, 0}); +# rec_rule.push_back({LLAMA_GRETYPE_END, 0}); +# add_rule(state, rec_rule_id, rec_rule); +# last_rec_rule_id = rec_rule_id; +# } +# if (n_opt > 0) { +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id}); +# } +# }; +# +# while (*pos) { +# if (*pos == '"') { // literal string +# pos++; +# last_sym_start = out_elements.size(); +# while (*pos != '"') { +# if (!*pos) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto char_pair = parse_char(pos); +# pos = char_pair.second; +# out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '[') { // char range(s) +# pos++; +# enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; +# if (*pos == '^') { # pos++; +# start_type = LLAMA_GRETYPE_CHAR_NOT; +# } +# last_sym_start = out_elements.size(); +# while (*pos != ']') { +# if (!*pos) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto char_pair = parse_char(pos); +# pos = char_pair.second; +# enum llama_gretype type = last_sym_start < out_elements.size() +# ? LLAMA_GRETYPE_CHAR_ALT +# : start_type; +# +# out_elements.push_back({type, char_pair.first}); +# if (pos[0] == '-' && pos[1] != ']') { +# if (!pos[1]) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto endchar_pair = parse_char(pos + 1); +# pos = endchar_pair.second; +# out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); +# } +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (is_word_char(*pos)) { // rule reference +# const char * name_end = parse_name(pos); +# uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); +# pos = parse_space(name_end, is_nested); +# last_sym_start = out_elements.size(); +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); +# } else if (*pos == '(') { // grouping +# // parse nested alternates into synthesized rule +# pos = parse_space(pos + 1, true); +# uint32_t sub_rule_id = generate_symbol_id(state, rule_name); +# pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); +# last_sym_start = out_elements.size(); +# // output reference to synthesized rule +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); +# if (*pos != ')') { +# throw std::runtime_error(std::string("expecting ')' at ") + pos); +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '.') { // any char +# last_sym_start = out_elements.size(); +# out_elements.push_back({LLAMA_GRETYPE_CHAR_ANY, 0}); +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '*') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(0, -1); +# } else if (*pos == '+') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(1, -1); +# } else if (*pos == '?') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(0, 1); +# } else if (*pos == '{') { +# pos = parse_space(pos + 1, is_nested); +# +# if (!is_digit_char(*pos)) { +# throw std::runtime_error(std::string("expecting an int at ") + pos); # } +# const char * int_end = parse_int(pos); +# int min_times = std::stoul(std::string(pos, int_end - pos)); +# pos = parse_space(int_end, is_nested); +# +# int max_times = -1; +# +# if (*pos == '}') { +# max_times = min_times; +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == ',') { +# pos = parse_space(pos + 1, is_nested); +# +# if (is_digit_char(*pos)) { +# const char * int_end = parse_int(pos); +# max_times = std::stoul(std::string(pos, int_end - pos)); +# pos = parse_space(int_end, is_nested); +# } +# +# if (*pos != '}') { +# throw std::runtime_error(std::string("expecting '}' at ") + pos); +# } +# pos = parse_space(pos + 1, is_nested); +# } else { +# throw std::runtime_error(std::string("expecting ',' at ") + pos); +# } +# handle_repetitions(min_times, max_times); # } else { -# pos++; +# break; # } # } # return pos; # } -def parse_space(src: const_char_p, newline_ok: bool) -> const_char_p: - pos = const_char_p(src) # type: const_char_p - while pos[0] in (" ", "\t", "#") or (newline_ok and pos[0] in ("\r", "\n")): - if pos[0] == "#": - while pos[0] is not None and pos[0] not in ("\r", "\n"): - pos += 1 - else: - pos += 1 - return pos +def parse_sequence(state: ParseState, src: str, rule_name: str, out_elements: typing.List[GrammarElement], is_nested: bool) -> str: + last_sym_start = len(out_elements) + pos = src + def handle_repetitions(min_times: int, max_times: int) -> None: + nonlocal state, src, rule_name, out_elements, is_nested, last_sym_start, pos -# const char * parse_sequence( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# std::vector & out_elements, -# bool is_nested) { -def parse_sequence( - state: parse_state, - src: const_char_p, - rule_name: str, - out_elements: std.vector[LlamaGrammarElement], - is_nested: bool, -) -> const_char_p: - # size_t last_sym_start = out_elements.size(); - # const char * pos = src; - last_sym_start = out_elements.size() # type: int - pos = const_char_p(src) # type: const_char_p - # while (*pos) { - while pos[0]: - # if (*pos == '"') { // literal string - # pos++; - # last_sym_start = out_elements.size(); - # while (*pos != '"') { - # auto char_pair = parse_char(pos); - # pos = char_pair.second; - # out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); - # } - # pos = parse_space(pos + 1, is_nested); - if pos[0] == '"': # literal string - pos += 1 - last_sym_start = out_elements.size() - while pos[0] != '"': - char_pair = parse_char(pos) # type: Tuple[int, const_char_p] - pos = char_pair[1] - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_CHAR, char_pair[0]) - ) - pos = parse_space(pos + 1, is_nested) - # } else if (*pos == '[') { // char range(s) - # pos++; - # enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; - elif pos[0] == "[": # char range(s) - pos += 1 - start_type = llama_gretype.LLAMA_GRETYPE_CHAR # type: llama_gretype - # if (*pos == '^') { - # pos++; - # start_type = LLAMA_GRETYPE_CHAR_NOT; - # } - # last_sym_start = out_elements.size(); + if last_sym_start == len(out_elements): + raise ValueError(f"expecting preceding item to */+/?/{{ at {pos}") + + previous_elements = out_elements[last_sym_start:] + if min_times == 0: + del out_elements[last_sym_start:] + else: + for i in range(1, min_times): + out_elements.extend(previous_elements) + + last_rec_rule_id = 0 + n_opt = 1 if max_times < 0 else max_times - min_times + + rec_rule = previous_elements[:] + for i in range(n_opt): + rec_rule = rec_rule[:len(previous_elements)] + rec_rule_id = generate_symbol_id(state, rule_name) + if i > 0 or max_times < 0: + rec_rule.append(GrammarElement(GrammarElementType.RULE_REF, rec_rule_id if max_times < 0 else last_rec_rule_id)) + rec_rule.append(GrammarElement(GrammarElementType.ALT, 0)) + rec_rule.append(GrammarElement(GrammarElementType.END, 0)) + add_rule(state, rec_rule_id, rec_rule) + last_rec_rule_id = rec_rule_id + if n_opt > 0: + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, last_rec_rule_id)) + + while pos: + if pos[0] == '"': + pos = pos[1:] + last_sym_start = len(out_elements) + while not pos.startswith('"'): + if not pos: + raise ValueError("unexpected end of input") + char, pos = parse_char(pos) + out_elements.append(GrammarElement(GrammarElementType.CHAR, char)) + pos = parse_space(pos[1:], is_nested) + elif pos[0] == "[": + pos = pos[1:] + start_type = GrammarElementType.CHAR if pos[0] == "^": - pos += 1 - start_type = llama_gretype.LLAMA_GRETYPE_CHAR_NOT - last_sym_start = out_elements.size() - # while (*pos != ']') { - # auto char_pair = parse_char(pos); - # pos = char_pair.second; - # enum llama_gretype type = last_sym_start < out_elements.size() - # ? LLAMA_GRETYPE_CHAR_ALT - # : start_type; - # out_elements.push_back({type, char_pair.first}); + pos = pos[1:] + start_type = GrammarElementType.CHAR_NOT + last_sym_start = len(out_elements) while pos[0] != "]": - char_pair = parse_char(pos) # type: Tuple[int, const_char_p] - pos = char_pair[1] - type = ( - llama_gretype.LLAMA_GRETYPE_CHAR_ALT - if last_sym_start < out_elements.size() - else start_type - ) # type: llama_gretype - out_elements.push_back(LlamaGrammarElement(type, char_pair[0])) - # if (pos[0] == '-' && pos[1] != ']') { - # auto endchar_pair = parse_char(pos + 1); - # pos = endchar_pair.second; - # out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); - # } - # } + if not pos: + raise ValueError("unexpected end of input") + char, pos = parse_char(pos) + type = GrammarElementType.CHAR_ALT if last_sym_start < len(out_elements) else start_type + out_elements.append(GrammarElement(type, char)) if pos[0] == "-" and pos[1] != "]": - endchar_pair = parse_char(pos + 1) # type: Tuple[int, const_char_p] - pos = endchar_pair[1] - out_elements.push_back( - LlamaGrammarElement( - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, - endchar_pair[0], - ) - ) - # pos = parse_space(pos + 1, is_nested); - pos = parse_space(pos + 1, is_nested) - # } else if (is_word_char(*pos)) { // rule reference - # const char * name_end = parse_name(pos); - # uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); - # pos = parse_space(name_end, is_nested); - # last_sym_start = out_elements.size(); - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); - elif is_word_char(pos[0]): # rule reference - name_end = parse_name(pos) # type: const_char_p - ref_rule_id = get_symbol_id(state, pos, name_end - pos) # type: int - pos = parse_space(name_end, is_nested) - last_sym_start = out_elements.size() - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, ref_rule_id) - ) - # } else if (*pos == '(') { // grouping - # // parse nested alternates into synthesized rule - # pos = parse_space(pos + 1, true); - # uint32_t sub_rule_id = generate_symbol_id(state, rule_name); - # pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); - # last_sym_start = out_elements.size(); - # // output reference to synthesized rule - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # if (*pos != ')') { - # throw std::runtime_error(std::string("expecting ')' at ") + pos); - # } - # pos = parse_space(pos + 1, is_nested); - elif pos[0] == "(": # grouping - # parse nested alternates into synthesized rule - pos = parse_space(pos + 1, True) - sub_rule_id = generate_symbol_id(state, rule_name) # type: int - pos = parse_alternates(state, pos, rule_name, sub_rule_id, True) - last_sym_start = out_elements.size() - # output reference to synthesized rule - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id) - ) + if not pos[1]: + raise ValueError("unexpected end of input") + endchar, pos = parse_char(pos[1:]) + out_elements.append(GrammarElement(GrammarElementType.CHAR_RNG_UPPER, endchar)) + pos = parse_space(pos[1:], is_nested) + elif pos and is_word_char(pos[0]): + name, rest = parse_name(pos) + ref_rule_id = get_symbol_id(state, name) + pos = parse_space(rest, is_nested) + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, ref_rule_id)) + elif pos.startswith("("): + pos = parse_space(pos[1:], newline_ok=True) + sub_rule_id = generate_symbol_id(state, rule_name) + pos = parse_alternates(state, pos, rule_name, sub_rule_id, is_nested=True) + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, sub_rule_id)) if pos[0] != ")": - raise RuntimeError("expecting ')' at " + str(pos)) - pos = parse_space(pos + 1, is_nested) - # } else if (*pos == '*' || *pos == '+' || *pos == '?') { // repetition operator - # if (last_sym_start == out_elements.size()) { - # throw std::runtime_error(std::string("expecting preceeding item to */+/? at ") + pos); - # } - elif pos[0] in ("*", "+", "?"): # repetition operator - if last_sym_start == out_elements.size(): - raise RuntimeError("expecting preceding item to */+/? at " + str(pos)) - # // apply transformation to previous symbol (last_sym_start to end) according to - # // rewrite rules: - # // S* --> S' ::= S S' | - # // S+ --> S' ::= S S' | S - # // S? --> S' ::= S | - # uint32_t sub_rule_id = generate_symbol_id(state, rule_name); - # std::vector sub_rule; - # // add preceding symbol to generated rule - # sub_rule.insert( - # sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end()); - sub_rule_id = generate_symbol_id(state, rule_name) # type: int - sub_rule = std.vector[ - LlamaGrammarElement - ]() # type: std.vector[LlamaGrammarElement] - sub_rule.insert( - sub_rule.end(), - out_elements.begin() + last_sym_start, - out_elements.end(), - ) - # if (*pos == '*' || *pos == '+') { - # // cause generated rule to recurse - # sub_rule.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # } - # // mark start of alternate def - # sub_rule.push_back({LLAMA_GRETYPE_ALT, 0}); - if pos[0] in ("*", "+"): - sub_rule.push_back( - LlamaGrammarElement( - llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id - ) - ) - sub_rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_ALT, 0)) - # if (*pos == '+') { - # // add preceding symbol as alternate only for '+' (otherwise empty) - # sub_rule.insert( - # sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end()); - # } - # sub_rule.push_back({LLAMA_GRETYPE_END, 0}); - # add_rule(state, sub_rule_id, sub_rule); - # // in original rule, replace previous symbol with reference to generated rule - # out_elements.resize(last_sym_start); - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # pos = parse_space(pos + 1, is_nested); - if pos[0] == "+": - # add preceding symbol as alternate only for '+' (otherwise empty) - sub_rule.insert( - sub_rule.end(), - out_elements.begin() + last_sym_start, - out_elements.end(), - ) - sub_rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_END, 0)) - add_rule(state, sub_rule_id, sub_rule) - # in original rule, replace previous symbol with reference to generated rule - out_elements.resize(last_sym_start) - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id) - ) - pos = parse_space(pos + 1, is_nested) - # } else { - # break; - # } + raise ValueError(f"expecting ')' at {pos}") + pos = parse_space(pos[1:], is_nested) + elif pos.startswith("."): + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.CHAR_ANY, 0)) + pos = parse_space(pos[1:], is_nested) + elif pos.startswith("*"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(0, -1) + elif pos.startswith("+"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(1, -1) + elif pos.startswith("?"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(0, 1) + elif pos.startswith("{"): + pos = parse_space(pos[1:], is_nested) + + if not is_digit_char(pos): + raise ValueError(f"expecting an int at {pos}") + min_times, pos = parse_int(pos) + pos = parse_space(pos, is_nested) + + max_times = -1 + + if pos[0] == "}": + max_times = min_times + pos = parse_space(pos[1:], is_nested) + elif pos[0] == ",": + pos = parse_space(pos[1:], is_nested) + + if is_digit_char(pos): + max_times, pos = parse_int(pos) + pos = parse_space(pos, is_nested) + + if pos[0] != "}": + raise ValueError("expecting '}' at {}".format(pos)) + + pos = parse_space(pos[1:], is_nested) + else: + raise ValueError(f"expecting ',' at {pos}") + handle_repetitions(min_times, max_times) else: break - # } - # return pos; - # } return pos @@ -919,39 +584,32 @@ def parse_sequence( # add_rule(state, rule_id, rule); # return pos; # } -def parse_alternates( - state: parse_state, - src: const_char_p, - rule_name: str, - rule_id: int, - is_nested: bool, -) -> const_char_p: - rule = std.vector() # type: std.vector[LlamaGrammarElement] - pos = parse_sequence(state, src, rule_name, rule, is_nested) # type: const_char_p - while pos[0] == "|": - rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_ALT, 0)) - pos = parse_space(pos + 1, True) +def parse_alternates(state: ParseState, src: str, rule_name: str, rule_id: int, is_nested: bool) -> str: + rule = [] + pos = parse_sequence(state, src, rule_name, rule, is_nested) + while pos.startswith("|"): + rule.append(GrammarElement(GrammarElementType.ALT, 0)) + pos = parse_space(pos[1:], newline_ok=True) pos = parse_sequence(state, pos, rule_name, rule, is_nested) - rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_END, 0)) + rule.append(GrammarElement(GrammarElementType.END, 0)) add_rule(state, rule_id, rule) return pos -# const char * parse_rule(parse_state & state, const char * src) { +# static const char * parse_rule(parse_state & state, const char * src) { # const char * name_end = parse_name(src); # const char * pos = parse_space(name_end, false); # size_t name_len = name_end - src; # uint32_t rule_id = get_symbol_id(state, src, name_len); # const std::string name(src, name_len); - +# # if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) { # throw std::runtime_error(std::string("expecting ::= at ") + pos); # } # pos = parse_space(pos + 3, true); - +# # pos = parse_alternates(state, pos, name, rule_id, false); - - +# # if (*pos == '\r') { # pos += pos[1] == '\n' ? 2 : 1; # } else if (*pos == '\n') { @@ -961,26 +619,26 @@ def parse_alternates( # } # return parse_space(pos, true); # } -def parse_rule(state: parse_state, src: const_char_p) -> const_char_p: - name_end = parse_name(src) # type: const_char_p - pos = parse_space(name_end, False) # type: const_char_p - name_len = name_end - src # type: int - rule_id = get_symbol_id(state, src, name_len) # type: int - name = std.string(src, name_len) # type: str - - if not (pos[0] == ":" and pos[1] == ":" and pos[2] == "="): - raise RuntimeError("expecting ::= at " + str(pos)) - - pos = parse_space(pos + 3, True) # type: const_char_p - pos = parse_alternates(state, pos, name, rule_id, False) # type: const_char_p - - if pos[0] == "\r": - pos += 2 if pos[1] == "\n" else 1 - elif pos[0] == "\n": - pos += 1 - elif pos[0]: - raise RuntimeError("expecting newline or end at " + str(pos)) - return parse_space(pos, True) +def parse_rule(state: ParseState, src: str) -> str: + pos = src + name, pos = parse_name(pos) + pos = parse_space(pos, newline_ok=False) + rule_id = get_symbol_id(state, name) + + if not pos.startswith("::="): + raise ValueError(f"expecting ::= at {pos}") + + pos = parse_space(pos[3:], newline_ok=True) + + pos = parse_alternates(state, pos, name, rule_id, is_nested=False) + + if pos.startswith("\r"): + pos = pos[2:] if pos[1] == "\n" else pos[1:] + elif pos.startswith("\n"): + pos = pos[1:] + elif pos: + raise ValueError(f"expecting newline or end at {pos}") + return parse_space(pos, newline_ok=True) # parse_state parse(const char * src) { @@ -990,204 +648,255 @@ def parse_rule(state: parse_state, src: const_char_p) -> const_char_p: # while (*pos) { # pos = parse_rule(state, pos); # } +# // Validate the state to ensure that all rules are defined +# for (const auto & rule : state.rules) { +# for (const auto & elem : rule) { +# if (elem.type == LLAMA_GRETYPE_RULE_REF) { +# // Ensure that the rule at that location exists +# if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) { +# // Get the name of the rule that is missing +# for (const auto & kv : state.symbol_ids) { +# if (kv.second == elem.value) { +# throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); +# } +# } +# } +# } +# } +# } # return state; # } catch (const std::exception & err) { # fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what()); # return parse_state(); # } # } -def parse(src: const_char_p) -> parse_state: - try: - state = parse_state() # type: parse_state - pos = parse_space(src, True) # type: const_char_p - while pos[0]: - pos = parse_rule(state, pos) - return state - except Exception as err: - print(f"{parse.__name__}: error parsing grammar: {err}") - return parse_state() - - -# void print_grammar_char(FILE * file, uint32_t c) { -# if (0x20 <= c && c <= 0x7f) { -# fprintf(file, "%c", static_cast(c)); -# } else { -# // cop out of encoding UTF-8 -# fprintf(file, "", c); -# } -# } -def print_grammar_char(file: TextIO, c: int) -> None: - if 0x20 <= c and c <= 0x7F: - file.write(chr(c)) - else: - # cop out of encoding UTF-8 - file.write(f"") - - -# bool is_char_element(llama_grammar_element elem) { +def parse(src: str) -> ParseState: + state = ParseState() + pos = src + pos = parse_space(pos, newline_ok=True) + while pos: + pos = parse_rule(state, pos) + # validate + for rule in state.rules: + for elem in rule: + if elem.type == GrammarElementType.RULE_REF: + if elem.value >= len(state.rules) or not state.rules[elem.value]: + for k, v in state.symbol_ids.items(): + if v == elem.value: + raise ValueError(f"Undefined rule identifier '{k}'") + return state + + +# static bool is_char_element(llama_grammar_element elem) { # switch (elem.type) { # case LLAMA_GRETYPE_CHAR: return true; # case LLAMA_GRETYPE_CHAR_NOT: return true; # case LLAMA_GRETYPE_CHAR_ALT: return true; # case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true; +# case LLAMA_GRETYPE_CHAR_ANY: return true; # default: return false; # } # } -def is_char_element(elem: LlamaGrammarElement) -> bool: +def is_char_element(elem: GrammarElement) -> bool: return elem.type in ( - llama_gretype.LLAMA_GRETYPE_CHAR, - llama_gretype.LLAMA_GRETYPE_CHAR_NOT, - llama_gretype.LLAMA_GRETYPE_CHAR_ALT, - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, + GrammarElementType.CHAR, + GrammarElementType.CHAR_NOT, + GrammarElementType.CHAR_ALT, + GrammarElementType.CHAR_RNG_UPPER, + GrammarElementType.CHAR_ANY ) -# void print_rule( +def print_grammar_char(file: typing.TextIO, c: int) -> None: + if 0x20 <= c <= 0x7f: + print(chr(c), end="", file=file) + else: + print(f"", end="", file=file) + + +# static void print_rule( # FILE * file, # uint32_t rule_id, # const std::vector & rule, # const std::map & symbol_id_names) { +# if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { +# throw std::runtime_error( +# "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); +# } +# fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); +# for (size_t i = 0, end = rule.size() - 1; i < end; i++) { +# llama_grammar_element elem = rule[i]; +# switch (elem.type) { +# case LLAMA_GRETYPE_END: +# throw std::runtime_error( +# "unexpected end of rule: " + std::to_string(rule_id) + "," + +# std::to_string(i)); +# case LLAMA_GRETYPE_ALT: +# fprintf(file, "| "); +# break; +# case LLAMA_GRETYPE_RULE_REF: +# fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); +# break; +# case LLAMA_GRETYPE_CHAR: +# fprintf(file, "["); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_NOT: +# fprintf(file, "[^"); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_RNG_UPPER: +# if (i == 0 || !is_char_element(rule[i - 1])) { +# throw std::runtime_error( +# "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + +# std::to_string(rule_id) + "," + std::to_string(i)); +# } +# fprintf(file, "-"); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_ALT: +# if (i == 0 || !is_char_element(rule[i - 1])) { +# throw std::runtime_error( +# "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + +# std::to_string(rule_id) + "," + std::to_string(i)); +# } +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_ANY: +# fprintf(file, "."); +# break; +# } +# if (is_char_element(elem)) { +# switch (rule[i + 1].type) { +# case LLAMA_GRETYPE_CHAR_ALT: +# case LLAMA_GRETYPE_CHAR_RNG_UPPER: +# case LLAMA_GRETYPE_CHAR_ANY: +# break; +# default: +# fprintf(file, "] "); +# } +# } +# } +# fprintf(file, "\n"); +# } def print_rule( - file: TextIO, + file: typing.TextIO, rule_id: int, - rule: std.vector[LlamaGrammarElement], - symbol_id_names: std.map[int, str], + rule: typing.List[GrammarElement], + symbol_id_names: typing.Dict[int, str], ) -> None: - # if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { - # throw std::runtime_error( - # "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); - # } - # fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); - if rule.empty() or rule.back().type != llama_gretype.LLAMA_GRETYPE_END: - raise RuntimeError( - "malformed rule, does not end with LLAMA_GRETYPE_END: " + str(rule_id) - ) - print(f"{symbol_id_names.at(rule_id)} ::=", file=file, end=" ") - # for (size_t i = 0, end = rule.size() - 1; i < end; i++) { - # llama_grammar_element elem = rule[i]; - # switch (elem.type) { - # case LLAMA_GRETYPE_END: - # throw std::runtime_error( - # "unexpected end of rule: " + std::to_string(rule_id) + "," + - # std::to_string(i)); - # case LLAMA_GRETYPE_ALT: - # fprintf(file, "| "); - # break; - # case LLAMA_GRETYPE_RULE_REF: - # fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); - # break; - # case LLAMA_GRETYPE_CHAR: - # fprintf(file, "["); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_NOT: - # fprintf(file, "[^"); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_RNG_UPPER: - # if (i == 0 || !is_char_element(rule[i - 1])) { - # throw std::runtime_error( - # "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + - # std::to_string(rule_id) + "," + std::to_string(i)); - # } - # fprintf(file, "-"); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_ALT: - # if (i == 0 || !is_char_element(rule[i - 1])) { - # throw std::runtime_error( - # "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + - # std::to_string(rule_id) + "," + std::to_string(i)); - # } - # print_grammar_char(file, elem.value); - # break; - # } + if not rule or rule[-1].type != GrammarElementType.END: + raise ValueError(f"malformed rule, does not end with LLAMA_GRETYPE_END: {rule_id}") + + print(f"{symbol_id_names[rule_id]} ::=", end=" ", file=file) + for i, elem in enumerate(rule[:-1]): - case = elem.type # type: llama_gretype - if case is llama_gretype.LLAMA_GRETYPE_END: - raise RuntimeError("unexpected end of rule: " + str(rule_id) + "," + str(i)) - elif case is llama_gretype.LLAMA_GRETYPE_ALT: - print("| ", file=file, end="") - elif case is llama_gretype.LLAMA_GRETYPE_RULE_REF: - print(f"{symbol_id_names.at(elem.value)} ", file=file, end="") - elif case is llama_gretype.LLAMA_GRETYPE_CHAR: - print("[", file=file, end="") + if elem.type == GrammarElementType.END: + raise ValueError(f"unexpected end of rule: {rule_id}, {i}") + if elem.type == GrammarElementType.ALT: + print("| ", end="", file=file) + elif elem.type == GrammarElementType.RULE_REF: + print(f"{symbol_id_names[elem.value]} ", end="", file=file) + elif elem.type == GrammarElementType.CHAR: + print("[", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_NOT: - print("[^", file=file, end="") + elif elem.type == GrammarElementType.CHAR_NOT: + print("[^", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER: + elif elem.type == GrammarElementType.CHAR_RNG_UPPER: if i == 0 or not is_char_element(rule[i - 1]): - raise RuntimeError( - "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " - + str(rule_id) - + "," - + str(i) - ) - print("-", file=file, end="") + raise ValueError(f"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: {rule_id}, {i}") + print(f"-", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_ALT: + elif elem.type == GrammarElementType.CHAR_ALT: if i == 0 or not is_char_element(rule[i - 1]): - raise RuntimeError( - "LLAMA_GRETYPE_CHAR_ALT without preceding char: " - + str(rule_id) - + "," - + str(i) - ) + raise ValueError(f"LLAMA_GRETYPE_CHAR_ALT without preceding char: {rule_id}, {i}") print_grammar_char(file, elem.value) - # if (is_char_element(elem)) { - # switch (rule[i + 1].type) { - # case LLAMA_GRETYPE_CHAR_ALT: - # case LLAMA_GRETYPE_CHAR_RNG_UPPER: - # break; - # default: - # fprintf(file, "] "); + elif elem.type == GrammarElementType.CHAR_ANY: + print(".", end="", file=file) if is_char_element(elem): - if rule[i + 1].type in ( - llama_gretype.LLAMA_GRETYPE_CHAR_ALT, - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, - ): - pass - else: - print("] ", file=file, end="") - # } - # } - # } - # fprintf(file, "\n"); - # } + if rule[i + 1].type in (GrammarElementType.CHAR_ALT, GrammarElementType.CHAR_RNG_UPPER, GrammarElementType.CHAR_ANY): + continue + print("] ", end="", file=file) print(file=file) -# void print_grammar(FILE * file, const parse_state & state) { -# try { -# std::map symbol_id_names; -# for (auto kv : state.symbol_ids) { -# symbol_id_names[kv.second] = kv.first; -# } -# for (size_t i = 0, end = state.rules.size(); i < end; i++) { -# // fprintf(file, "%zu: ", i); -# // print_rule_binary(file, state.rules[i]); -# print_rule(file, i, state.rules[i], symbol_id_names); -# // fprintf(file, "\n"); -# } -# } catch (const std::exception & err) { -# fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what()); -# } -# } -def print_grammar(file: TextIO, state: parse_state) -> None: +def print_grammar(file: typing.TextIO, state: ParseState) -> None: try: - symbol_id_names = std.map() # type: std.map[int, str] - for kv in state.symbol_ids.items(): - symbol_id_names[kv[1]] = kv[0] - + symbol_id_names = {v: k for k, v in state.symbol_ids.items()} for i, rule in enumerate(state.rules): print_rule(file, i, rule, symbol_id_names) except Exception as err: - print( - f"{print_grammar.__name__}: error printing grammar: {err}", - file=sys.stderr, + print(f"\nerror printing grammar: {err}", file=file) + raise err + + +class LlamaGrammar: + def __init__(self, parse_state: ParseState): + self.parse_state = parse_state + + self._grammar_rules = parse_state.rules + self._n_rules = len(self._grammar_rules) + self._start_rule_index = parse_state.symbol_ids["root"] + + self._element_lists = [ + [ + llama_cpp.llama_grammar_element(ctypes.c_int(elem.type), ctypes.c_uint32(elem.value)) + for elem in subvector + ] + for subvector in self._grammar_rules + ] + + # Step 2: Convert each list to llama_grammar_element array and get pointer + self._element_arrays = [ + (llama_cpp.llama_grammar_element * len(sublist))(*sublist) + for sublist in self._element_lists + ] + + # Step 3: Get pointer of each array + self._element_array_pointers = [ + ctypes.cast(subarray, llama_cpp.llama_grammar_element_p) for subarray in self._element_arrays + ] + + # Step 4: Make array of these pointers and get its pointer + self._rules = (llama_cpp.llama_grammar_element_p * len(self._element_array_pointers))( + *self._element_array_pointers ) + self.grammar = None + self._init_grammar() + + + def _init_grammar(self): + grammar = llama_cpp.llama_grammar_init( + self._rules, ctypes.c_size_t(self._n_rules), ctypes.c_size_t(self._start_rule_index) + ) + + if grammar is None: + raise ValueError("Failed to create grammar") + + self.grammar = grammar + + def __del__(self): + if self.grammar is not None: + llama_cpp.llama_grammar_free(self.grammar) + self.grammar = None + + def reset(self): + if self.grammar is not None: + llama_cpp.llama_grammar_free(self.grammar) + self._init_grammar() + + @classmethod + def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": + parsed_grammar = parse(grammar) + print_grammar(file=sys.stdout, state=parsed_grammar) + return cls(parsed_grammar) + + @classmethod + def from_json_schema(cls, json_schema: str, verbose: bool = True) -> "LlamaGrammar": + return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose) + """llama.cpp gbnf rules from vendor/llama.cpp/grammars""" @@ -1358,12 +1067,13 @@ def print_grammar(file: TextIO, state: parse_state) -> None: string ::= "\"" ( [^"\\\x7F\x00-\x1F] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes + "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes )* "\"" ws -number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws +number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws -ws ::= ([ \t\n] ws)? +# Optional space: by convention, applied in this grammar after literal chars when allowed +ws ::= | " " | "\n" [ \t]{0,20} """ LIST_GBNF = r""" From 18f58fe39fceac821b9cf88ae40355292ade2a8e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 6 Aug 2024 20:21:18 -0400 Subject: [PATCH 023/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++-- vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d598cf9f1..643a321f5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -944,7 +944,7 @@ class llama_context_params(ctypes.Structure): # int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() # enum llama_ftype ftype; // quantize to this llama_ftype # enum ggml_type output_tensor_type; // output tensor type -# enum ggml_type token_embedding_type; // itoken embeddings tensor type +# enum ggml_type token_embedding_type; // token embeddings tensor type # bool allow_requantize; // allow quantizing non-f32/f16 tensors # bool quantize_output_tensor; // quantize output.weight # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored @@ -960,7 +960,7 @@ class llama_model_quantize_params(ctypes.Structure): nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() ftype (int): quantize to this llama_ftype output_tensor_type (int): output tensor type - token_embedding_type (int): itoken embeddings tensor type + token_embedding_type (int): token embeddings tensor type allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 398ede5ef..725e3d943 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 398ede5efeb07b9adf9fbda7ea63f630d476a792 +Subproject commit 725e3d94379d5b619c027347308bccf2e0ead89f From ce6466f0eeb5c28d90b3cd26fb5a12959f45138c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 6 Aug 2024 20:23:57 -0400 Subject: [PATCH 024/155] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc25442b..6630f1958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.86] + +- feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 +- feat: Ported back new grammar changes from C++ to Python implementation by @ExtReMLapin in (#1637) +- fix: llama_grammar_accept_token arg order by @tc-wolf in (#1649) + ## [0.2.85] - feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 97c8d8174..7b672c046 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.85" +__version__ = "0.2.86" From 198f47dc1bd202fd2b71b29e041a9f33fe40bfad Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:33:05 -0400 Subject: [PATCH 025/155] feat(ci): Re-build wheel index automatically when releases are created --- .github/workflows/generate-index-from-release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 500c4613c..662cb6f65 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -1,9 +1,11 @@ name: Wheels Index on: - # Trigger on any new release - release: - types: [published] + # Trigger on new release + workflow_run: + workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)"] + types: + - completed # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From a07b337241264ab3b5cadf14b8fd81d1729b86e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:33:45 -0400 Subject: [PATCH 026/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 725e3d943..be55695ef 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 725e3d94379d5b619c027347308bccf2e0ead89f +Subproject commit be55695eff44784a141a863f273661a6bce63dfc From 9cad5714ae6e7c250af8d0bbb179f631368c928b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:50:37 -0400 Subject: [PATCH 027/155] fix: Include all llama.cpp source files and subdirectories --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8345cb1f0..7aebb74f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ wheel.packages = ["llama_cpp"] cmake.verbose = true cmake.minimum-version = "3.21" minimum-version = "0.5.1" -sdist.include = [".git", "vendor/llama.cpp/.git"] +sdist.include = [".git", "vendor/llama.cpp/*"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 84321162f8cad3e1caf552d7f0b18bc869e2cebf Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:52:11 -0400 Subject: [PATCH 028/155] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6630f1958..e5c7f01c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.87] + +- feat: Update llama.cpp to ggerganov/llama.cpp@be55695eff44784a141a863f273661a6bce63dfc +- fix: Include all llama.cpp source files and subdirectories by @abetlen in 9cad5714ae6e7c250af8d0bbb179f631368c928b +- feat(ci): Re-build wheel index automatically when releases are created by @abetlen in 198f47dc1bd202fd2b71b29e041a9f33fe40bfad + ## [0.2.86] - feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 7b672c046..37d91cd54 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.86" +__version__ = "0.2.87" From e966f3bcc1296fae7ba0d490eb12bf8ce3ced9fb Mon Sep 17 00:00:00 2001 From: Xu Song Date: Wed, 7 Aug 2024 22:41:39 +0800 Subject: [PATCH 029/155] feat: Add more detailed log for prefix-match (#1659) Co-authored-by: Andrei --- llama_cpp/llama.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 0cb5ca2fc..3fa3b563a 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -777,11 +777,12 @@ def generate( else: break if longest_prefix > 0: - if self.verbose: - print("Llama.generate: prefix-match hit", file=sys.stderr) reset = False tokens = tokens[longest_prefix:] self.n_tokens = longest_prefix + if self.verbose: + print(f"Llama.generate: {longest_prefix} prefix-match hit, " + f"remaining {len(tokens)} prompt tokens to eval", file=sys.stderr) # Reset the model state if reset: From 131db407700df32efd0ced55c75c0bd44def9b1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:42:27 -0400 Subject: [PATCH 030/155] chore(deps): bump pypa/cibuildwheel from 2.19.2 to 2.20.0 (#1657) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.19.2 to 2.20.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.2...v2.20.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index eeeda803d..785f38847 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 47c1c3cb5..dd8b9c8fc 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -30,7 +30,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From 5e39a854631c0a8442374d47acb90310fa751501 Mon Sep 17 00:00:00 2001 From: Benedikt Heidrich Date: Wed, 7 Aug 2024 16:52:50 +0200 Subject: [PATCH 031/155] feat: Enable recursive search of HFFS.ls when using `from_pretrained` (#1656) Enable rercursive ls for hffs. Otherwise models in subfolder wouldn't be found. Co-authored-by: Andrei --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3fa3b563a..49608908b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2160,7 +2160,7 @@ def from_pretrained( files = [ file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id) + for file in hffs.ls(repo_id, recursive=True)) ] # split each file into repo_id, subfolder, filename From c5de5d3e882c6ec5abf346d34cf329b4a8a35916 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:32:33 -0400 Subject: [PATCH 032/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index be55695ef..afd27f01f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit be55695eff44784a141a863f273661a6bce63dfc +Subproject commit afd27f01fe832ece3d07ef03b7d34a9e80c4a895 From 0998ea0deea076a547d54bd598d6b413b588ee2b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:34:20 -0400 Subject: [PATCH 033/155] fix: grammar prints on each call. Closes #1666 --- llama_cpp/llama_grammar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index e60817e1d..bc2a9cc45 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -890,7 +890,8 @@ def reset(self): @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": parsed_grammar = parse(grammar) - print_grammar(file=sys.stdout, state=parsed_grammar) + if verbose: + print_grammar(file=sys.stdout, state=parsed_grammar) return cls(parsed_grammar) @classmethod From 7aaf7015800b98fd246b4c747e59d766318dec71 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:35:31 -0400 Subject: [PATCH 034/155] fix: typo --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 49608908b..a37a62aef 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2160,7 +2160,7 @@ def from_pretrained( files = [ file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id, recursive=True)) + for file in hffs.ls(repo_id, recursive=True) ] # split each file into repo_id, subfolder, filename From 45de9d5fa6ca445a6e0ef23ef5e7d15348808ab2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 10 Aug 2024 18:33:33 -0400 Subject: [PATCH 035/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 8 ++++++++ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 643a321f5..1a8424aa1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1505,6 +1505,14 @@ def llama_model_has_encoder(model: llama_model_p, /) -> bool: ... +# // Returns true if the model contains a decoder that requires llama_decode() call +# LLAMA_API bool llama_model_has_decoder(const struct llama_model * model); +@ctypes_function("llama_model_has_decoder", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_has_decoder(model: llama_model_p, /) -> bool: + """Returns true if the model contains a decoder that requires llama_decode() call""" + ... + + # // For encoder-decoder models, this function returns id of the token that must be provided # // to the decoder to start generating output sequence. For other models, it returns -1. # LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index afd27f01f..6e02327e8 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit afd27f01fe832ece3d07ef03b7d34a9e80c4a895 +Subproject commit 6e02327e8b7837358e0406bf90a4632e18e27846 From 4244151d7a27a29d8ac56dfbdc7ae10b68defaa5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 12 Aug 2024 02:29:30 -0400 Subject: [PATCH 036/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6e02327e8..4134999e0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6e02327e8b7837358e0406bf90a4632e18e27846 +Subproject commit 4134999e01f31256b15342b41c4de9e2477c4a6c From 95a1533172b7d31bf2fa9a54b4f2a9776fbd4c45 Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:12:54 +0200 Subject: [PATCH 037/155] fix: Added back from_file method to LlamaGrammar (#1673) --- llama_cpp/llama_grammar.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index bc2a9cc45..4cd52c2d5 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1,6 +1,7 @@ """Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.""" # flake8: noqa +from pathlib import Path import sys import ctypes import enum @@ -893,6 +894,23 @@ def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": if verbose: print_grammar(file=sys.stdout, state=parsed_grammar) return cls(parsed_grammar) + + @classmethod + def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": + try: + with open(file) as f: + grammar = f.read() + except Exception as err: + raise Exception( + f"{cls.from_file.__name__}: error reading grammar file: {err}" + ) + + if grammar: + return cls.from_string(grammar, verbose=verbose) + + raise ValueError( + f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty" + ) @classmethod def from_json_schema(cls, json_schema: str, verbose: bool = True) -> "LlamaGrammar": From 9bab46f643570a29d356337eb5ab995af8862aea Mon Sep 17 00:00:00 2001 From: Laurent Sorber Date: Mon, 12 Aug 2024 20:19:26 +0200 Subject: [PATCH 038/155] fix: only print 'cache saved' in verbose mode (#1668) Co-authored-by: Andrei --- llama_cpp/llama.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index a37a62aef..b780ef6c6 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1523,7 +1523,8 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache save", file=sys.stderr) self.cache[prompt_tokens + completion_tokens] = self.save_state() - print("Llama._create_completion: cache saved", file=sys.stderr) + if self.verbose: + print("Llama._create_completion: cache saved", file=sys.stderr) return if self.cache: From 8ed663b867e6ba56dc03096984675dd35667f2b7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 12 Aug 2024 14:19:56 -0400 Subject: [PATCH 039/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4134999e0..fc4ca27b2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4134999e01f31256b15342b41c4de9e2477c4a6c +Subproject commit fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 From fc19cc783802ed95eb4c8be358537b12c1fec78f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 13 Aug 2024 05:22:04 -0400 Subject: [PATCH 040/155] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c7f01c1..f7738e1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.88] + +- feat: Update llama.cpp to ggerganov/llama.cpp@fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 +- fix: only print 'cache saved' in verbose mode by @lsorber in #1668 +- fix: Added back from_file method to LlamaGrammar by @ExtReMLapin in #1673 +- fix: grammar prints on each call by @abetlen in 0998ea0deea076a547d54bd598d6b413b588ee2b +- feat: Enable recursive search of HFFS.ls when using from_pretrained by @benHeidabetlen in #1656 +- feat: Add more detailed log for prefix-match by @xu-song in #1659 + ## [0.2.87] - feat: Update llama.cpp to ggerganov/llama.cpp@be55695eff44784a141a863f273661a6bce63dfc diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 37d91cd54..0030e8f7e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.87" +__version__ = "0.2.88" From 63d65acabc7d7402cdb85926a70b2ef81b472ac4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 15 Aug 2024 14:46:46 -0400 Subject: [PATCH 041/155] feat: Update llama.cpp --- llama_cpp/_internals.py | 8 ++++---- llama_cpp/llama.py | 4 ++-- llama_cpp/llama_cpp.py | 26 ++++++++++++-------------- vendor/llama.cpp | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 357023e3c..6dae88c8f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -179,11 +179,11 @@ def token_eot(self) -> int: assert self.model is not None return llama_cpp.llama_token_eot(self.model) - def add_bos_token(self) -> int: + def add_bos_token(self) -> bool: assert self.model is not None return llama_cpp.llama_add_bos_token(self.model) - def add_eos_token(self) -> int: + def add_eos_token(self) -> bool: assert self.model is not None return llama_cpp.llama_add_eos_token(self.model) @@ -691,8 +691,8 @@ def _detokenize_bpe(model: _LlamaModel, tokens: List[int]) -> str: def _should_add_bos(model: _LlamaModel) -> bool: assert model.model is not None add_bos = llama_cpp.llama_add_bos_token(model.model) - if add_bos != -1: - return add_bos != 0 + if add_bos: + return add_bos else: return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b780ef6c6..29674c6d8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1058,13 +1058,13 @@ def _create_completion( if ( (isinstance(prompt, list) and suffix is None) - or self._model.add_bos_token() == 0 + or not self._model.add_bos_token() or bos_tokens[:1] == [-1] ): bos_tokens = [] if (isinstance(prompt, list) and suffix is None) or ( - self._model.add_eos_token() != 1 and sep_token_id == -1 + not self._model.add_eos_token() and sep_token_id == -1 ): eos_tokens = [] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1a8424aa1..00064f324 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -314,6 +314,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, # LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, # LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -338,6 +340,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 +LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 +LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 # // note: these values should be synchronized with ggml_rope @@ -345,13 +349,11 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # enum llama_rope_type { # LLAMA_ROPE_TYPE_NONE = -1, # LLAMA_ROPE_TYPE_NORM = 0, -# LLAMA_ROPE_TYPE_NEOX = 2, -# LLAMA_ROPE_TYPE_GLM = 4, +# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, # }; LLAMA_ROPE_TYPE_NONE = -1 LLAMA_ROPE_TYPE_NORM = 0 -LLAMA_ROPE_TYPE_NEOX = 2 -LLAMA_ROPE_TYPE_GLM = 4 +LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file @@ -2741,19 +2743,15 @@ def llama_token_nl(model: llama_model_p, /) -> int: ... -# // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); -@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32) -def llama_add_bos_token(model: llama_model_p, /) -> int: - """Returns -1 if unknown, 1 for true or 0 for false.""" +# LLAMA_API bool llama_add_bos_token(const struct llama_model * model); +@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) +def llama_add_bos_token(model: llama_model_p, /) -> bool: ... -# // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); -@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32) -def llama_add_eos_token(model: llama_model_p, /) -> int: - """Returns -1 if unknown, 1 for true or 0 for false.""" +# LLAMA_API bool llama_add_eos_token(const struct llama_model * model); +@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) +def llama_add_eos_token(model: llama_model_p, /) -> bool: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fc4ca27b2..4b9afbbe9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 +Subproject commit 4b9afbbe9037f8a2d659097c0c7d9fce32c6494c From 78e35c4998f3209cb70c188a47cffd17c1f12bce Mon Sep 17 00:00:00 2001 From: Junpei Kawamoto Date: Thu, 15 Aug 2024 15:00:46 -0600 Subject: [PATCH 042/155] fix: missing dependencies for test (#1680) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7aebb74f5..ce50c673f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,10 @@ test = [ "pytest>=7.4.0", "httpx>=0.24.1", "scipy>=1.10", + "fastapi>=0.100.0", + "sse-starlette>=1.6.1", + "starlette-context>=0.3.6,<0.4", + "pydantic-settings>=2.0.1", ] dev = [ "black>=23.3.0", From 3c7501b35755205d1df70181cd46e769ec2e9790 Mon Sep 17 00:00:00 2001 From: Junpei Kawamoto Date: Thu, 15 Aug 2024 15:01:18 -0600 Subject: [PATCH 043/155] fix: Llama.close didn't free lora adapter (#1679) --- llama_cpp/llama.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 29674c6d8..9b1a3d263 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -198,6 +198,7 @@ def __init__( A Llama instance. """ self.verbose = verbose + self._stack = contextlib.ExitStack() set_verbose(verbose) @@ -365,8 +366,6 @@ def __init__( if not os.path.exists(model_path): raise ValueError(f"Model path does not exist: {model_path}") - self._stack = contextlib.ExitStack() - self._model = self._stack.enter_context( contextlib.closing( _LlamaModel( @@ -420,6 +419,15 @@ def __init__( raise RuntimeError( f"Failed to initialize LoRA adapter from lora path: {self.lora_path}" ) + + def free_lora_adapter(): + if self._lora_adapter is None: + return + llama_cpp.llama_lora_adapter_free(self._lora_adapter) + self._lora_adapter = None + + self._stack.callback(free_lora_adapter) + assert self._ctx.ctx is not None if llama_cpp.llama_lora_adapter_set( self._ctx.ctx, self._lora_adapter, self.lora_scale @@ -2085,14 +2093,9 @@ def pooling_type(self) -> str: def close(self) -> None: """Explicitly free the model from memory.""" - if hasattr(self,'_stack'): - if self._stack is not None: - self._stack.close() + self._stack.close() def __del__(self) -> None: - if hasattr(self,'_lora_adapter'): - if self._lora_adapter is not None: - llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod From 7bf07ec51873ab96ac326bac126044039768ef4b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 16 Aug 2024 14:43:25 -0400 Subject: [PATCH 044/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 00064f324..476713aee 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -316,6 +316,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, # LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -342,6 +343,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 +LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4b9afbbe9..8b3befc0e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4b9afbbe9037f8a2d659097c0c7d9fce32c6494c +Subproject commit 8b3befc0e2ed8fb18b903735831496b8b0c80949 From a2ba73165223f8946f3e1180e3cb67d7d9236da2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 19 Aug 2024 13:36:31 -0400 Subject: [PATCH 045/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8b3befc0e..cfac111e2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8b3befc0e2ed8fb18b903735831496b8b0c80949 +Subproject commit cfac111e2b3953cdb6b0126e67a2487687646971 From d7328efabf841a750259dc42089efece13efc0f5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 19 Aug 2024 13:36:49 -0400 Subject: [PATCH 046/155] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7738e1a9..063fe722b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.89] + +- feat: Update llama.cpp to ggerganov/llama.cpp@cfac111e2b3953cdb6b0126e67a2487687646971 +- fix: Llama.close didn't free lora adapter by @jkawamoto in #1679 +- fix: missing dependencies for test by @jkawamoto in #1680 + ## [0.2.88] - feat: Update llama.cpp to ggerganov/llama.cpp@fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 0030e8f7e..e3cece2f6 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.88" +__version__ = "0.2.89" From a20f13fc3b08e7c2163d84d5da51da78ec9f48c3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 21 Aug 2024 10:11:21 -0400 Subject: [PATCH 047/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index cfac111e2..fc54ef0d1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit cfac111e2b3953cdb6b0126e67a2487687646971 +Subproject commit fc54ef0d1c138133a01933296d50a36a1ab64735 From 259ee151da9a569f58f6d4979e97cfd5d5bc3ecd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 22 Aug 2024 09:17:45 -0400 Subject: [PATCH 048/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 8 ++++++++ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 476713aee..b35f56a79 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1530,6 +1530,14 @@ def llama_model_decoder_start_token(model: llama_model_p, /) -> int: ... +# // Returns true if the model is recurrent (like Mamba, RWKV, etc.) +# LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model); +@ctypes_function("llama_model_is_recurrent", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_is_recurrent(model: llama_model_p, /) -> bool: + """Returns true if the model is recurrent (like Mamba, RWKV, etc.)""" + ... + + # // Returns 0 on success # LLAMA_API uint32_t llama_model_quantize( # const char * fname_inp, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fc54ef0d1..1731d4238 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fc54ef0d1c138133a01933296d50a36a1ab64735 +Subproject commit 1731d4238f9e4f925a750810e7f5480827c66dcf From 82ae7f93716c47b4ed0a809f280d5352e2fcdea9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 28 Aug 2024 00:47:48 -0400 Subject: [PATCH 049/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1731d4238..20f1789df 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1731d4238f9e4f925a750810e7f5480827c66dcf +Subproject commit 20f1789dfb4e535d64ba2f523c64929e7891f428 From f70df824985d875226793b94dacc0c302a4256b2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:05:32 -0400 Subject: [PATCH 050/155] feat: Add MiniCPMv26 chat handler. --- llama_cpp/llama_chat_format.py | 92 +++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index ea8d07feb..3c7330296 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2707,6 +2707,31 @@ def last_image_embed_free(): def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) + def _embed_image_bytes(self, image_bytes: bytes, n_threads_batch: int = 1): + if ( + self._last_image_embed is not None + and self._last_image_hash is not None + and hash(image_bytes) == self._last_image_hash + ): + return self._last_image_embed + with suppress_stdout_stderr(disable=self.verbose): + # Free the previous image embed + if self._last_image_embed is not None: + self._llava_cpp.llava_image_embed_free(self._last_image_embed) + self._last_image_embed = None + self._last_image_hash = None + embed = self._llava_cpp.llava_image_embed_make_with_bytes( + self.clip_ctx, + n_threads_batch, + (ctypes.c_uint8 * len(image_bytes)).from_buffer( + bytearray(image_bytes) + ), + len(image_bytes), + ) + self._last_image_embed = embed + self._last_image_hash = hash(image_bytes) + return embed + def __call__( self, *, @@ -2769,30 +2794,9 @@ def __call__( ) split_text = self.split_text_on_image_urls(text, image_urls) - def embed_image_bytes(image_bytes: bytes): - if ( - self._last_image_embed is not None - and self._last_image_hash is not None - and hash(image_bytes) == self._last_image_hash - ): - return self._last_image_embed - with suppress_stdout_stderr(disable=self.verbose): - # Free the previous image embed - if self._last_image_embed is not None: - self._llava_cpp.llava_image_embed_free(self._last_image_embed) - self._last_image_embed = None - self._last_image_hash = None - embed = self._llava_cpp.llava_image_embed_make_with_bytes( - self.clip_ctx, - llama.context_params.n_threads_batch, - (ctypes.c_uint8 * len(image_bytes)).from_buffer( - bytearray(image_bytes) - ), - len(image_bytes), - ) - self._last_image_embed = embed - self._last_image_hash = hash(image_bytes) - return embed + if self.verbose: + print(text, file=sys.stderr) + # Evaluate prompt llama.reset() @@ -2809,7 +2813,7 @@ def embed_image_bytes(image_bytes: bytes): llama.eval(tokens) else: image_bytes = self.load_image(value) - embed = embed_image_bytes(image_bytes) + embed = self._embed_image_bytes(image_bytes, llama.context_params.n_threads_batch) if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): raise ValueError( f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}" @@ -3308,6 +3312,44 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): Llama3VisionAlpha = Llama3VisionAlphaChatHandler +class MiniCPMv26(Llava15ChatHandler): + DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." + + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if loop.first and messages[0]['role'] != 'system' %}" + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "{% endif %}" + "<|im_start|>{{ message['role'] }}\n" + "{% if message['content'] is iterable %}" + "{% for content in message['content'] %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + + "{% for content in message['content'] %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% if message['content'] is string %}" + "{{ message['content'] }}" + "{% endif %}" + "<|im_end|>\n" + "{% endfor %}" + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama.Llama, From e251a0bd7d05f993f19e79e3f8328e4d3b41110a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:26:34 -0400 Subject: [PATCH 051/155] fix: Update name to MiniCPMv26ChatHandler --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 3c7330296..21091bf50 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3312,7 +3312,7 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): Llama3VisionAlpha = Llama3VisionAlphaChatHandler -class MiniCPMv26(Llava15ChatHandler): +class MiniCPMv26ChatHandler(Llava15ChatHandler): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." CHAT_FORMAT = ( From c68e7fbedfff8c385167a3d5232ad6a4c12597f1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:51:25 -0400 Subject: [PATCH 052/155] fix: pull all gh releases for self-hosted python index --- .../generate-index-from-release.yaml | 3 + scripts/get-releases.sh | 47 +++++++ scripts/releases-to-pep-503.sh | 122 ++++++++++++------ 3 files changed, 133 insertions(+), 39 deletions(-) create mode 100755 scripts/get-releases.sh diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 662cb6f65..51961088d 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -35,7 +35,10 @@ jobs: - name: Setup Pages uses: actions/configure-pages@v5 - name: Build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + ./scripts/get-releases.sh ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$' ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' diff --git a/scripts/get-releases.sh b/scripts/get-releases.sh new file mode 100755 index 000000000..4c904da78 --- /dev/null +++ b/scripts/get-releases.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Function to get all releases +get_all_releases() { + local page=1 + local per_page=100 + local releases="" + local new_releases + + # Prepare headers + local headers=(-H "Accept: application/vnd.github.v3+json") + if [ -n "$GITHUB_TOKEN" ]; then + headers+=(-H "Authorization: Bearer $GITHUB_TOKEN") + fi + + while true; do + response=$(curl -s "${headers[@]}" \ + "https://api.github.com/repos/abetlen/llama-cpp-python/releases?page=$page&per_page=$per_page") + + # Check if the response is valid JSON + if ! echo "$response" | jq empty > /dev/null 2>&1; then + echo "Error: Invalid response from GitHub API" >&2 + echo "Response: $response" >&2 + return 1 + fi + + new_releases=$(echo "$response" | jq -r '.[].tag_name') + if [ -z "$new_releases" ]; then + break + fi + releases="$releases $new_releases" + ((page++)) + done + + echo $releases +} + +# Get all releases and save to file +releases=$(get_all_releases) +if [ $? -ne 0 ]; then + echo "Failed to fetch releases. Please check your internet connection and try again later." >&2 + exit 1 +fi + +echo "$releases" | tr ' ' '\n' > all_releases.txt + +echo "All releases have been saved to all_releases.txt" diff --git a/scripts/releases-to-pep-503.sh b/scripts/releases-to-pep-503.sh index 44fbbf3cf..71910efcb 100755 --- a/scripts/releases-to-pep-503.sh +++ b/scripts/releases-to-pep-503.sh @@ -1,60 +1,104 @@ #!/bin/bash +# Enable exit on error +set -e + +# Function for logging +log_error() { + echo "ERROR: $1" >&2 +} + +log_info() { + echo "INFO: $1" +} + # Get output directory or default to index/whl/cpu output_dir=${1:-"index/whl/cpu"} -# Create output directory -mkdir -p $output_dir - -# Change to output directory -pushd $output_dir +# Get pattern from second arg or default to valid python package version pattern +pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"} -# Create an index html file -echo "" > index.html -echo "" >> index.html -echo " " >> index.html -echo " " >> index.html -echo " llama-cpp-python" >> index.html -echo "
" >> index.html -echo " " >> index.html -echo "" >> index.html -echo "" >> index.html +# Get the current directory (where the script is run from) +current_dir="$(pwd)" -# Create llama-cpp-python directory -mkdir -p llama-cpp-python +# Check if all_releases.txt exists +if [ ! -f "$current_dir/all_releases.txt" ]; then + log_error "all_releases.txt not found in the current directory." + exit 1 +fi -# Change to llama-cpp-python directory -pushd llama-cpp-python +# Create output directory +mkdir -p "$output_dir" # Create an index html file -echo "" > index.html -echo "" >> index.html -echo " " >> index.html -echo "

Links for llama-cpp-python

" >> index.html +cat << EOF > "$output_dir/index.html" + + + + + llama-cpp-python +
+ + -# Get all releases -releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name) +EOF -# Get pattern from second arg or default to valid python package version pattern -pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"} +# Create llama-cpp-python directory +mkdir -p "$output_dir/llama-cpp-python" + +# Create an index html file in llama-cpp-python directory +cat << EOF > "$output_dir/llama-cpp-python/index.html" + + + +

Links for llama-cpp-python

+EOF # Filter releases by pattern -releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern) +releases=$(grep -E "$pattern" "$current_dir/all_releases.txt") + +# Prepare curl headers +headers=('--header' 'Accept: application/vnd.github.v3+json') +if [ -n "$GITHUB_TOKEN" ]; then + headers+=('--header' "authorization: Bearer $GITHUB_TOKEN") +fi +headers+=('--header' 'content-type: application/json') # For each release, get all assets for release in $releases; do - assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets) + log_info "Processing release: $release" + response=$(curl -s "${headers[@]}" \ + "https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release") + + if [ -z "$response" ]; then + log_error "Empty response from GitHub API for release $release" + continue + fi + + if ! echo "$response" | jq -e '.assets' > /dev/null 2>&1; then + log_error "Invalid or unexpected response from GitHub API for release $release" + log_error "Response: $response" + continue + fi + # Get release version from release ie v0.1.0-cu121 -> v0.1.0 - release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") - echo "

$release_version

" >> index.html - for asset in $(echo $assets | jq -r .[].browser_download_url); do - if [[ $asset == *".whl" ]]; then - echo " $asset" >> index.html - echo "
" >> index.html - fi + release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") + echo "

$release_version

" >> "$output_dir/llama-cpp-python/index.html" + + wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url') + if [ -z "$wheel_urls" ]; then + log_error "No wheel files found for release $release" + continue + fi + + echo "$wheel_urls" | while read -r asset; do + echo " $asset" >> "$output_dir/llama-cpp-python/index.html" + echo "
" >> "$output_dir/llama-cpp-python/index.html" done done -echo " " >> index.html -echo "" >> index.html -echo "" >> index.html +echo " " >> "$output_dir/llama-cpp-python/index.html" +echo "" >> "$output_dir/llama-cpp-python/index.html" +echo "" >> "$output_dir/llama-cpp-python/index.html" + +log_info "Index generation complete. Output directory: $output_dir" From 97d527eec2b879f38e9eed599f8c7d2ae3a0b23e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:19:13 -0400 Subject: [PATCH 053/155] feat: Add server chat_format minicpm-v-2.6 for MiniCPMv26ChatHandler --- llama_cpp/server/model.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index c486f8885..901998094 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -157,6 +157,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Llama3VisionAlpha( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "minicpm-v-2.6": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.MiniCPMv26ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.MiniCPMv26ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From b570fd38c1f5e2b29b801d16e8b556409f25527b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:37:08 -0400 Subject: [PATCH 054/155] =?UTF-8?q?docs:=20Add=20project=20icon=20courtesy?= =?UTF-8?q?=20of=20=F0=9F=A4=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ docs/icon.svg | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 docs/icon.svg diff --git a/README.md b/README.md index b0dfdd5b5..b8065149c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![icon.svg](./docs/icon.svg) + # 🦙 Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) [![Documentation Status](https://readthedocs.org/projects/llama-cpp-python/badge/?version=latest)](https://llama-cpp-python.readthedocs.io/en/latest/?badge=latest) diff --git a/docs/icon.svg b/docs/icon.svg new file mode 100644 index 000000000..9f2d08753 --- /dev/null +++ b/docs/icon.svg @@ -0,0 +1,5 @@ + + + + + From cbbfad45929ebbc90d79bfaaa1dbdf2825a2db3f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:40:06 -0400 Subject: [PATCH 055/155] docs: center icon and resize --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8065149c..ea544d4d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -![icon.svg](./docs/icon.svg) +

+ +

-# 🦙 Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) +# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) [![Documentation Status](https://readthedocs.org/projects/llama-cpp-python/badge/?version=latest)](https://llama-cpp-python.readthedocs.io/en/latest/?badge=latest) [![Tests](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml) From ad2deafa8d615e9eaf0f8c3976e465fb1a3ea15f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:51:39 -0400 Subject: [PATCH 056/155] docs: Add MiniCPM-V-2.6 to multi-modal model list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ea544d4d1..b63efa3c8 100644 --- a/README.md +++ b/README.md @@ -503,6 +503,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [moondream2](https://huggingface.co/vikhyatk/moondream2) | `MoondreamChatHandler` | `moondream2` | | [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanollavaChatHandler` | `nanollava` | | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | +| [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. From 332720d00332334f42576bebf3a8a3da1a27bfaf Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:56:37 -0400 Subject: [PATCH 057/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 20f1789df..1d1ccce67 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 20f1789dfb4e535d64ba2f523c64929e7891f428 +Subproject commit 1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 From 077ecb6771fbd373d5507444f6e0b6e9bb7cf4e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:57:36 -0400 Subject: [PATCH 058/155] chore: Bump version --- llama_cpp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index e3cece2f6..db1f1d0fa 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.89" +__version__ = "0.2.90" From 45001ac550dc6360b4809dd7564d3304d4ad8fa6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 10:12:23 -0400 Subject: [PATCH 059/155] misc(fix): Update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 063fe722b..f01b1b9bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.90] + +- feat: Update llama.cpp to ggerganov/llama.cpp@1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 +- feat: Add support for `MiniCPMv26ChatHandler` and `minicpm-v-26` in server by @abetlen in f70df824985d875226793b94dacc0c302a4256b2 + ## [0.2.89] - feat: Update llama.cpp to ggerganov/llama.cpp@cfac111e2b3953cdb6b0126e67a2487687646971 From 4b1e3642daa06412008ab2e7366aad09281a70bb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 10:25:21 -0400 Subject: [PATCH 060/155] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b63efa3c8..29807bbdf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From 8b853c0f256c7318938b3894721637174938af17 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 12:41:05 -0400 Subject: [PATCH 061/155] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29807bbdf..a8f14e898 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From 9cba3b8198c5f8d3c3bd12b5ddf5c32838c25040 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 12:41:28 -0400 Subject: [PATCH 062/155] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8f14e898..cbf1c4158 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From d981d32dac3d35b39cc164a10b4edc11cc5b89db Mon Sep 17 00:00:00 2001 From: benniekiss <63211101+benniekiss@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:04:00 -0400 Subject: [PATCH 063/155] feat: Enable detokenizing special tokens with `special=True` (#1596) * enable detokenizing special tokens * enable skipping_special_tokens in hf_tokenizer detokenize() * process prev_tokens * fix doc strings * Revert changes to LlamaTokenizer prev_tokens and set special to False by default --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 9 ++++++--- llama_cpp/llama_tokenizer.py | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 9b1a3d263..45cbd7bca 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -578,6 +578,8 @@ def tokenize( Args: text: The utf-8 encoded string to tokenize. + add_bos: Whether to add a beginning of sequence token. + special: Whether to tokenize special tokens. Raises: RuntimeError: If the tokenization failed. @@ -588,18 +590,19 @@ def tokenize( return self.tokenizer_.tokenize(text, add_bos, special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: """Detokenize a list of tokens. Args: tokens: The list of tokens to detokenize. - prev_tokens: The list of previous tokens. Offset mapping will be performed if provided + prev_tokens: The list of previous tokens. Offset mapping will be performed if provided. + special: Whether to detokenize special tokens. Returns: The detokenized string. """ - return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens) + return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens, special=special) def set_cache(self, cache: Optional[BaseLlamaCache]): """Set the cache. diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 029bf2acc..2e7590d14 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -19,20 +19,22 @@ def tokenize( """Tokenize the text into tokens. Args: - text: The text to tokenize. + text: The utf-8 encoded string to tokenize. add_bos: Whether to add a beginning of sequence token. - special: Whether to tokenize text literally or as special tokens.""" + special: Whether to tokenize special tokens. + """ raise NotImplementedError @abc.abstractmethod def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: """Detokenize the tokens into text. Args: - tokens: The tokens to detokenize. - prev_tokens: If tokens is a continuation of a previous sequence, the previous tokens. + tokens: The list of tokens to detokenize. + prev_tokens: The list of previous tokens. Offset mapping will be performed if provided. + special: Whether to detokenize special tokens. """ raise NotImplementedError @@ -47,9 +49,9 @@ def tokenize( return self._model.tokenize(text, add_bos=add_bos, special=special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: - return self._model.detokenize(tokens) + return self._model.detokenize(tokens, special=special) def encode( self, text: str, add_bos: bool = True, special: bool = True @@ -78,18 +80,19 @@ def tokenize( ) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: + skip_special_tokens = not special if prev_tokens is not None: - text = self.hf_tokenizer.decode(prev_tokens + tokens).encode( + text = self.hf_tokenizer.decode(prev_tokens + tokens, skip_special_tokens=skip_special_tokens).encode( "utf-8", errors="ignore" ) - prev_text = self.hf_tokenizer.decode(prev_tokens).encode( + prev_text = self.hf_tokenizer.decode(prev_tokens, skip_special_tokens=skip_special_tokens).encode( "utf-8", errors="ignore" ) return text[len(prev_text) :] else: - return self.hf_tokenizer.decode(tokens).encode("utf-8", errors="ignore") + return self.hf_tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens).encode("utf-8", errors="ignore") @classmethod def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer": From 98eb092d3c6e7c142c4ba2faaca6c091718abbb3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 30 Aug 2024 00:14:49 -0400 Subject: [PATCH 064/155] fix: Use system message in og qwen format. Closes #1697 --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 21091bf50..dfb0af65e 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1009,7 +1009,7 @@ def format_qwen( **kwargs: Any, ) -> ChatFormatterResponse: _roles = dict(user="<|im_start|>user", assistant="<|im_start|>assistant") - system_message = "You are a helpful assistant." + system_message = _get_system_message(messages) or "You are a helpful assistant." system_template = "<|im_start|>system\n{system_message}" system_message = system_template.format(system_message=system_message) _messages = _map_roles(messages, _roles) From dcb0d0cf2aae4b5f4b30da49106f69c6bd5346a8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 30 Aug 2024 00:33:58 -0400 Subject: [PATCH 065/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 36 +++++++++++++++++++++++------------- vendor/llama.cpp | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index b35f56a79..d2e35a2d1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -795,8 +795,8 @@ class llama_model_params(ctypes.Structure): # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_threads; // number of threads to use for generation -# uint32_t n_threads_batch; // number of threads to use for batch processing +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id @@ -901,8 +901,8 @@ class llama_context_params(ctypes.Structure): ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), - ("n_threads", ctypes.c_uint32), - ("n_threads_batch", ctypes.c_uint32), + ("n_threads", ctypes.c_int32), + ("n_threads_batch", ctypes.c_int32), ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), ("attention_type", ctypes.c_int), @@ -1197,6 +1197,16 @@ def llama_numa_init(numa: int, /): ... +# // Optional: an auto threadpool gets created in ggml if not passed explicitly +# LLAMA_API void llama_attach_threadpool( +# struct llama_context * ctx, +# ggml_threadpool_t threadpool, +# ggml_threadpool_t threadpool_batch); + + +# LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + # // Call once at the end of the program - currently only used for MPI # LLAMA_API void llama_backend_free(void); @ctypes_function( @@ -2478,20 +2488,20 @@ def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: # // Set the number of threads used for decoding # // n_threads is the number of threads used for generation (single token) # // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) -# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); +# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); @ctypes_function( "llama_set_n_threads", [ llama_context_p_ctypes, - ctypes.c_uint32, - ctypes.c_uint32, + ctypes.c_int32, + ctypes.c_int32, ], None, ) def llama_set_n_threads( ctx: llama_context_p, - n_threads: Union[ctypes.c_uint32, int], - n_threads_batch: Union[ctypes.c_uint32, int], + n_threads: Union[ctypes.c_int32, int], + n_threads_batch: Union[ctypes.c_int32, int], /, ): """Set the number of threads used for decoding @@ -2502,16 +2512,16 @@ def llama_set_n_threads( # // Get the number of threads used for generation of a single token. -# LLAMA_API uint32_t llama_n_threads(struct llama_context * ctx); -@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_uint32) +# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); +@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads(ctx: llama_context_p, /) -> int: """Get the number of threads used for generation of a single token""" ... # // Get the number of threads used for prompt and batch processing (multiple token). -# LLAMA_API uint32_t llama_n_threads_batch(struct llama_context * ctx); -@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_uint32) +# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); +@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads_batch(ctx: llama_context_p, /) -> int: """Get the number of threads used for prompt and batch processing (multiple token)""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1d1ccce67..42c76d135 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 +Subproject commit 42c76d1358021ccdbe8ba89109c143dd7ae166df From 9769e5719ad45d4be6bc3ebe01d0f568d3d5001e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 31 Aug 2024 08:59:41 -0400 Subject: [PATCH 066/155] feat: Update llama.cpp --- llama_cpp/llama.py | 2 +- llama_cpp/llama_cpp.py | 6 +++--- vendor/llama.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 45cbd7bca..7fac936d1 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -153,7 +153,7 @@ def __init__( model_path: Path to the model. n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. - main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored + main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. rpc_servers: Comma separated list of RPC servers to use for offloading vocab_only: Only load the vocabulary no weights. diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d2e35a2d1..c1ba14fe5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -711,9 +711,9 @@ class llama_model_kv_override(ctypes.Structure): # enum llama_split_mode split_mode; // how to split the model across multiple GPUs # // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_LAYER: ignored +# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model +# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results +# // LLAMA_SPLIT_MODE_LAYER: ignored # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 42c76d135..a47667cff 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 42c76d1358021ccdbe8ba89109c143dd7ae166df +Subproject commit a47667cff41f5a198eb791974e0afcc1cddd3229 From c3fc80a2cf5e88360c982a3187751083e881bd16 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 1 Sep 2024 21:06:18 -0400 Subject: [PATCH 067/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 3 +++ vendor/llama.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c1ba14fe5..501323774 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -276,6 +276,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE # LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece # LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram +# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization # }; LLAMA_VOCAB_TYPE_NONE = 0 """For models without vocab""" @@ -287,6 +288,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa """BERT tokenizer based on WordPiece""" LLAMA_VOCAB_TYPE_UGM = 4 """T5 tokenizer based on Unigram""" +LLAMA_VOCAB_TYPE_RWKV = 5 +"""RWKV tokenizer based on greedy tokenization""" # // pre-tokenization types diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a47667cff..8f1d81a0b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a47667cff41f5a198eb791974e0afcc1cddd3229 +Subproject commit 8f1d81a0b6f50b9bad72db0b6fcd299ad9ecd48c From 9497bcd6da7c1675f1309892fd3206e44c38940f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 4 Sep 2024 23:01:55 -0400 Subject: [PATCH 068/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8f1d81a0b..581c30518 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8f1d81a0b6f50b9bad72db0b6fcd299ad9ecd48c +Subproject commit 581c305186a0ff93f360346c57e21fe16e967bb7 From c032fc65b0873337ed39e5d63e15468a5d797646 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Sep 2024 04:12:29 -0400 Subject: [PATCH 069/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++++ vendor/llama.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 501323774..ecaf2369c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -443,6 +443,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -479,6 +481,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 +LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 +LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 581c30518..8ebe8ddeb 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 581c305186a0ff93f360346c57e21fe16e967bb7 +Subproject commit 8ebe8ddebd68526757c631cd019de009697c63c2 From e529940f45d42ed8aa31334123b8d66bc67b0e78 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 19 Sep 2024 01:22:05 +0200 Subject: [PATCH 070/155] feat(ci): Speed up CI workflows using `uv`, add support for CUDA 12.5 wheels * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * revert * Bump pyhton from 3.8 to 3.9 * Remove python 3.8 * Remove Python 3.7 and 3.8 deprecated * Bump python from 3.8 to 3.9 * Add python 3.9 * Add python 3.9, remove macos-11 deprecated, add macos-14 * Bump python 3.8 to 3.9 * Add python 3.13 * Add python 3.13 * python 3.13 remove * remove python 3.13 * remove python 3.8 * Bump macos-13 to macos-14 * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update generate-index-from-release.yaml Add avx, avx2 and avx512 * Update test.yaml * Update test-pypi.yaml * Update publish.yaml * Update publish-to-test.yaml * Update build-wheels-cuda.yaml Cuda with AVX2 by default * Update build-wheels-cuda.yaml * remove DEPRECATED 32 bits * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml Upgrade matrix os to latest version * Update build-wheels-metal.yaml * Update build-wheels-cuda.yaml * Update test.yaml * Update test-pypi.yaml * Update test.yaml Add cache: 'pip' * Update publish-to-test.yaml * Update build-wheels-metal.yaml Add cache: 'pip' * Update build-wheels-cuda.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml remove x86_64 * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * revert * Remove cpu variants * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update publish-to-test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update test-pypi.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update test-pypi.yaml * Update publish-to-test.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish-to-test.yaml * Update publish.yaml * Update test-pypi.yaml * Update test.yaml * Update test.yaml * Update build-and-release.yaml * Update publish-to-test.yaml * Update build-wheels-metal.yaml * Update test-pypi.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update publish-to-test.yaml * Update test-pypi.yaml * Update test.yaml * Update build-wheels-cuda.yaml * Update generate-index-from-release.yaml * Update README.md * Update README.md * Update test.yaml --------- Co-authored-by: Andrei Betlen --- .github/workflows/build-and-release.yaml | 44 +++++++++-- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 17 ++++- .../generate-index-from-release.yaml | 1 + .github/workflows/publish-to-test.yaml | 24 +++++- .github/workflows/publish.yaml | 23 +++++- .github/workflows/test-pypi.yaml | 59 +++++++++++++-- .github/workflows/test.yaml | 75 +++++++++++++++++-- README.md | 3 +- 9 files changed, 217 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 785f38847..ab8ff4e1c 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -21,12 +21,25 @@ jobs: # Used to host cibuildwheel - uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - - name: Install dependencies + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | python -m pip install --upgrade pip - python -m pip install -e .[all] + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd - name: Build wheels uses: pypa/cibuildwheel@v2.20.0 @@ -79,16 +92,33 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - uses: actions/setup-python@v5 with: - python-version: "3.8" - - name: Install dependencies + python-version: "3.9" + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - uses: actions/upload-artifact@v4 with: name: sdist diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 0733a68c5..b1e5fc1a9 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.0") 'releasetag' = @("basic") } diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index dd8b9c8fc..551820822 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -23,11 +23,24 @@ jobs: with: python-version: "3.12" cache: 'pip' + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash - - name: Install dependencies + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install -e .[all] + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd - name: Build wheels uses: pypa/cibuildwheel@v2.20.0 diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 51961088d..8d80e7e47 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,6 +44,7 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index 19613233b..de3ae42aa 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -19,24 +19,42 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.11" cache: 'pip' + - name: Append Dev Version to __version__ run: | DEV_VERSION=${{ github.event.inputs.dev_version }} CURRENT_VERSION=$(awk -F= '/__version__ =/ {print $2}' llama_cpp/__init__.py | tr -d ' "') NEW_VERSION="${CURRENT_VERSION}.dev${DEV_VERSION}" sed -i 's/__version__ = \".*\"/__version__ = \"'"${NEW_VERSION}"'\"/' llama_cpp/__init__.py - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - name: Publish to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index c6abb43b3..67816ea03 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,17 +13,34 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.9" - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - name: Publish distribution to PyPI # TODO: move to tag based releases # if: startsWith(github.ref, 'refs/tags') diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index d7131956d..335033bba 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -16,10 +16,25 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" @@ -37,10 +52,25 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" @@ -57,11 +87,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install dependencies + cache: 'pip' + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 78f0b4983..9de9c9229 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,15 +20,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install .[all] -v + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -44,15 +60,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install .[all] -v + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -68,15 +100,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | python -m pip install --upgrade pip - python -m pip install .[all] --verbose + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -90,11 +138,24 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: python-version: "3.9" - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose diff --git a/README.md b/README.md index cbf1c4158..12b4b127d 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: -- CUDA Version is 12.1, 12.2, 12.3, or 12.4 +- CUDA Version is 12.1, 12.2, 12.3, 12.4 or 12.5 - Python Version is 3.10, 3.11 or 3.12 ```bash @@ -138,6 +138,7 @@ Where `` is one of the following: - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 - `cu124`: CUDA 12.4 +- `cu124`: CUDA 12.5 For example, to install the CUDA 12.1 wheel: From a4e1451abe02c02fc9c831abb591d0559a54b849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:26:51 -0400 Subject: [PATCH 071/155] chore(deps): bump pypa/cibuildwheel from 2.20.0 to 2.21.1 (#1743) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.20.0 to 2.21.1. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.20.0...v2.21.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index ab8ff4e1c..3f85a30fe 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -42,7 +42,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -69,7 +69,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 551820822..8e6fb5cb7 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -43,7 +43,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From f8fcb3ea3424bcfba3a5437626a994771a02324b Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 18 Sep 2024 20:00:19 -0400 Subject: [PATCH 072/155] feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test --- .github/workflows/test.yaml | 81 ++- llama_cpp/_internals.py | 361 +++++------ llama_cpp/_logger.py | 19 +- llama_cpp/llama.py | 227 ++++--- llama_cpp/llama_cpp.py | 1178 ++++++++++++++++------------------- llama_cpp/llama_grammar.py | 881 +------------------------- pyproject.toml | 1 + tests/test_llama.py | 358 ++++------- tests/test_llama_grammar.py | 10 +- vendor/llama.cpp | 2 +- 10 files changed, 1035 insertions(+), 2083 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9de9c9229..f24f5b7e1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,4 @@ name: Tests - on: pull_request: branches: @@ -8,14 +7,34 @@ on: branches: - main +env: + REPO_ID: Qwen/Qwen2-0.5B-Instruct-GGUF + MODEL_FILE: qwen2-0_5b-instruct-q8_0.gguf + jobs: - build-linux: + download-model: + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install huggingface-hub + run: pip install huggingface-hub + - name: Download model + run: huggingface-cli download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} + - name: Cache model + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + build-linux: + needs: download-model runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -26,36 +45,35 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd - + shell: cmd - name: Test with pytest run: | python -m pytest build-windows: - + needs: download-model runs-on: windows-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -66,19 +84,23 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv @@ -90,12 +112,11 @@ jobs: python -m pytest build-macos: - + needs: download-model runs-on: macos-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -106,34 +127,36 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd + shell: cmd - name: Test with pytest run: | python -m pytest - build-macos-metal: - + needs: download-model runs-on: macos-latest - steps: - uses: actions/checkout@v4 with: @@ -144,18 +167,22 @@ jobs: with: python-version: "3.9" + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 6dae88c8f..6475695c6 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -6,6 +6,7 @@ from typing import ( Dict, List, + Tuple, Optional, Sequence, ) @@ -25,7 +26,7 @@ # Python wrappers over llama.h structs -class _LlamaModel: +class LlamaModel: """Intermediate Python wrapper for a llama.cpp llama_model. NOTE: For stability it's recommended you use the Llama class instead.""" @@ -41,19 +42,21 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.model = None + model = None if not os.path.exists(path_model): raise ValueError(f"Model path does not exist: {path_model}") with suppress_stdout_stderr(disable=verbose): - self.model = llama_cpp.llama_load_model_from_file( + model = llama_cpp.llama_load_model_from_file( self.path_model.encode("utf-8"), self.params ) - if self.model is None: + if model is None: raise ValueError(f"Failed to load model from file: {path_model}") + self.model = model + def free_model(): if self.model is None: return @@ -69,128 +72,84 @@ def __del__(self): self.close() def vocab_type(self) -> int: - assert self.model is not None return llama_cpp.llama_vocab_type(self.model) def n_vocab(self) -> int: - assert self.model is not None return llama_cpp.llama_n_vocab(self.model) def n_ctx_train(self) -> int: - assert self.model is not None return llama_cpp.llama_n_ctx_train(self.model) def n_embd(self) -> int: - assert self.model is not None return llama_cpp.llama_n_embd(self.model) def rope_freq_scale_train(self) -> float: - assert self.model is not None return llama_cpp.llama_rope_freq_scale_train(self.model) def desc(self) -> str: - assert self.model is not None buf = ctypes.create_string_buffer(1024) llama_cpp.llama_model_desc(self.model, buf, 1024) return buf.value.decode("utf-8") def size(self) -> int: - assert self.model is not None return llama_cpp.llama_model_size(self.model) def n_params(self) -> int: - assert self.model is not None return llama_cpp.llama_model_n_params(self.model) def get_tensor(self, name: str) -> ctypes.c_void_p: - assert self.model is not None return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) - def apply_lora_from_file( - self, - lora_path: str, - scale: float, - path_base_model: Optional[str], - n_threads: int, - ): - assert self.model is not None - return llama_cpp.llama_model_apply_lora_from_file( - self.model, - lora_path.encode("utf-8"), - scale, - ( - path_base_model.encode("utf-8") - if path_base_model is not None - else ctypes.c_char_p(0) - ), - n_threads, - ) # Vocab def token_get_text(self, token: int) -> str: - # TODO: Fix - assert self.model is not None return llama_cpp.llama_token_get_text(self.model, token).decode("utf-8") def token_get_score(self, token: int) -> float: - assert self.model is not None return llama_cpp.llama_token_get_score(self.model, token) def token_get_attr(self, token: int) -> int: - assert self.model is not None return llama_cpp.llama_token_get_attr(self.model, token) # Special tokens def token_bos(self) -> int: - assert self.model is not None return llama_cpp.llama_token_bos(self.model) def token_eos(self) -> int: - assert self.model is not None return llama_cpp.llama_token_eos(self.model) def token_cls(self) -> int: - assert self.model is not None return llama_cpp.llama_token_cls(self.model) def token_sep(self) -> int: - assert self.model is not None return llama_cpp.llama_token_sep(self.model) def token_nl(self) -> int: - assert self.model is not None return llama_cpp.llama_token_nl(self.model) def token_prefix(self) -> int: - assert self.model is not None return llama_cpp.llama_token_prefix(self.model) def token_middle(self) -> int: - assert self.model is not None return llama_cpp.llama_token_middle(self.model) def token_suffix(self) -> int: - assert self.model is not None return llama_cpp.llama_token_suffix(self.model) def token_eot(self) -> int: - assert self.model is not None return llama_cpp.llama_token_eot(self.model) def add_bos_token(self) -> bool: - assert self.model is not None return llama_cpp.llama_add_bos_token(self.model) def add_eos_token(self) -> bool: - assert self.model is not None return llama_cpp.llama_add_eos_token(self.model) # Tokenization def tokenize(self, text: bytes, add_bos: bool, special: bool): - assert self.model is not None n_ctx = self.n_ctx_train() tokens = (llama_cpp.llama_token * n_ctx)() n_tokens = llama_cpp.llama_tokenize( @@ -209,13 +168,11 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): return list(tokens[:n_tokens]) def token_to_piece(self, token: int, special: bool = False) -> bytes: - assert self.model is not None buf = ctypes.create_string_buffer(32) llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special) return bytes(buf) def detokenize(self, tokens: List[int], special: bool = False) -> bytes: - assert self.model is not None output = b"" size = 32 buffer = (ctypes.c_char * size)() @@ -235,7 +192,6 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: # Extra def metadata(self) -> Dict[str, str]: - assert self.model is not None metadata: Dict[str, str] = {} buffer_size = 1024 buffer = ctypes.create_string_buffer(buffer_size) @@ -272,14 +228,14 @@ def default_params(): return llama_cpp.llama_model_default_params() -class _LlamaContext: +class LlamaContext: """Intermediate Python wrapper for a llama.cpp llama_context. NOTE: For stability it's recommended you use the Llama class instead.""" def __init__( self, *, - model: _LlamaModel, + model: LlamaModel, params: llama_cpp.llama_context_params, verbose: bool = True, ): @@ -288,15 +244,13 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.ctx = None + ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) - assert self.model.model is not None - - self.ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) - - if self.ctx is None: + if ctx is None: raise ValueError("Failed to create llama_context") + self.ctx = ctx + def free_ctx(): if self.ctx is None: return @@ -312,35 +266,27 @@ def __del__(self): self.close() def n_ctx(self) -> int: - assert self.ctx is not None return llama_cpp.llama_n_ctx(self.ctx) def pooling_type(self) -> int: - assert self.ctx is not None return llama_cpp.llama_pooling_type(self.ctx) def kv_cache_clear(self): - assert self.ctx is not None llama_cpp.llama_kv_cache_clear(self.ctx) def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_rm(self.ctx, seq_id, p0, p1) def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_cp(self.ctx, seq_id_src, seq_id_dst, p0, p1) def kv_cache_seq_keep(self, seq_id: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_keep(self.ctx, seq_id) def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_add(self.ctx, seq_id, p0, p1, shift) def get_state_size(self) -> int: - assert self.ctx is not None return llama_cpp.llama_get_state_size(self.ctx) # TODO: copy_state_data @@ -351,9 +297,7 @@ def get_state_size(self) -> int: # TODO: llama_save_session_file - def decode(self, batch: "_LlamaBatch"): - assert self.ctx is not None - assert batch.batch is not None + def decode(self, batch: LlamaBatch): return_code = llama_cpp.llama_decode( self.ctx, batch.batch, @@ -362,25 +306,21 @@ def decode(self, batch: "_LlamaBatch"): raise RuntimeError(f"llama_decode returned {return_code}") def set_n_threads(self, n_threads: int, n_threads_batch: int): - assert self.ctx is not None llama_cpp.llama_set_n_threads(self.ctx, n_threads, n_threads_batch) def get_logits(self): - assert self.ctx is not None return llama_cpp.llama_get_logits(self.ctx) def get_logits_ith(self, i: int): - assert self.ctx is not None return llama_cpp.llama_get_logits_ith(self.ctx, i) def get_embeddings(self): - assert self.ctx is not None return llama_cpp.llama_get_embeddings(self.ctx) # Sampling functions def set_rng_seed(self, seed: int): - assert self.ctx is not None + # TODO: Fix llama_cpp.llama_set_rng_seed(self.ctx, seed) def sample_repetition_penalties( @@ -392,7 +332,6 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - assert self.ctx is not None llama_cpp.llama_sample_repetition_penalties( self.ctx, llama_cpp.byref(candidates.candidates), @@ -404,26 +343,22 @@ def sample_repetition_penalties( ) def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - assert self.ctx is not None llama_cpp.llama_sample_softmax( self.ctx, llama_cpp.byref(candidates.candidates), ) def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_top_k( self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep ) def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_top_p( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_min_p( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) @@ -431,7 +366,6 @@ def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: i def sample_tail_free( self, candidates: "_LlamaTokenDataArray", z: float, min_keep: int ): - assert self.ctx is not None llama_cpp.llama_sample_tail_free( self.ctx, llama_cpp.byref(candidates.candidates), z, min_keep ) @@ -439,20 +373,16 @@ def sample_tail_free( def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - assert self.ctx is not None llama_cpp.llama_sample_typical( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): - assert self.ctx is not None llama_cpp.llama_sample_temp( self.ctx, llama_cpp.byref(candidates.candidates), temp ) def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - assert self.ctx is not None - assert grammar.grammar is not None llama_cpp.llama_sample_grammar( self.ctx, llama_cpp.byref(candidates.candidates), @@ -467,7 +397,6 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_mirostat( self.ctx, llama_cpp.byref(candidates.candidates), @@ -484,7 +413,6 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_mirostat_v2( self.ctx, llama_cpp.byref(candidates.candidates), @@ -494,14 +422,12 @@ def sample_token_mirostat_v2( ) def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_greedy( self.ctx, llama_cpp.byref(candidates.candidates), ) def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token( self.ctx, llama_cpp.byref(candidates.candidates), @@ -509,17 +435,13 @@ def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - assert self.ctx is not None - assert grammar.grammar is not None llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): - assert self.ctx is not None - llama_cpp.llama_reset_timings(self.ctx) + llama_cpp.llama_perf_context_reset(self.ctx) def print_timings(self): - assert self.ctx is not None - llama_cpp.llama_print_timings(self.ctx) + llama_cpp.llama_perf_context_print(self.ctx) # Utility functions @staticmethod @@ -528,7 +450,7 @@ def default_params(): return llama_cpp.llama_context_default_params() -class _LlamaBatch: +class LlamaBatch: def __init__( self, *, n_tokens: int, embd: int, n_seq_max: int, verbose: bool = True ): @@ -538,11 +460,15 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.batch = None - self.batch = llama_cpp.llama_batch_init( + batch = llama_cpp.llama_batch_init( self._n_tokens, self.embd, self.n_seq_max ) + if batch is None: + raise ValueError("Failed to create llama_batch") + + self.batch = batch + def free_batch(): if self.batch is None: return @@ -558,15 +484,12 @@ def __del__(self): self.close() def n_tokens(self) -> int: - assert self.batch is not None return self.batch.n_tokens def reset(self): - assert self.batch is not None self.batch.n_tokens = 0 def set_batch(self, batch: Sequence[int], n_past: int, logits_all: bool): - assert self.batch is not None n_tokens = len(batch) self.batch.n_tokens = n_tokens for i in range(n_tokens): @@ -578,7 +501,6 @@ def set_batch(self, batch: Sequence[int], n_past: int, logits_all: bool): self.batch.logits[n_tokens - 1] = True def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): - assert self.batch is not None n_tokens = len(batch) n_tokens0 = self.batch.n_tokens self.batch.n_tokens += n_tokens @@ -592,7 +514,7 @@ def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): self.batch.logits[n_tokens - 1] = True -class _LlamaTokenDataArray: +class LlamaTokenDataArray: def __init__(self, *, n_vocab: int): self.n_vocab = n_vocab self.candidates_data = np.recarray( @@ -617,90 +539,9 @@ def copy_logits(self, logits: npt.NDArray[np.single]): self.candidates.size = self.n_vocab -# Python wrappers over common/common -def _tokenize(model: _LlamaModel, text: str, add_bos: bool, special: bool) -> list[int]: - assert model.model is not None - n_tokens = len(text) + 1 if add_bos else len(text) - result = (llama_cpp.llama_token * n_tokens)() - n_tokens = llama_cpp.llama_tokenize( - model.model, - text.encode("utf-8"), - len(text), - result, - n_tokens, - add_bos, - special, - ) - if n_tokens < 0: - result = (llama_cpp.llama_token * -n_tokens)() - check = llama_cpp.llama_tokenize( - model.model, - text.encode("utf-8"), - len(text), - result, - len(result), - add_bos, - special, - ) - if check != -n_tokens: - raise RuntimeError(f'Failed to tokenize: text="{text}" n_tokens={n_tokens}') - else: - result = result[:n_tokens] - return list(result) - - -def _token_to_piece(model: _LlamaModel, token: int, special: bool = False) -> str: - assert model.model is not None - result = (ctypes.c_char * 8)(0) - n_tokens = llama_cpp.llama_token_to_piece( - model.model, token, result, 0, len(result), special - ) - if n_tokens < 0: - result = (ctypes.c_char * -n_tokens)(0) - check = llama_cpp.llama_token_to_piece( - model.model, token, result, 0, len(result), special - ) - if check != -n_tokens: - raise RuntimeError(f"Failed to get piece: token={token}") - else: - result = result[:n_tokens] - return bytes(result).decode("utf-8") - - -def _detokenize_spm(model: _LlamaModel, tokens: List[int]) -> str: - bos_id = model.token_bos() - result = "" - for i, token in enumerate(tokens): - piece = _token_to_piece(model, token) - if ( - (tokens[0] == bos_id and i == 1) or (tokens[0] != bos_id and i == 0) - ) and piece[0] == " ": - piece = piece[1:] - result += piece - return result - - -def _detokenize_bpe(model: _LlamaModel, tokens: List[int]) -> str: - result = "" - for token in tokens: - piece = _token_to_piece(model, token) - result += piece - return result - - -def _should_add_bos(model: _LlamaModel) -> bool: - assert model.model is not None - add_bos = llama_cpp.llama_add_bos_token(model.model) - if add_bos: - return add_bos - else: - return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM - - # Embedding functions - -def _normalize_embedding(embedding): +def normalize_embedding(embedding): norm = float(np.linalg.norm(embedding)) if norm == 0.0: return embedding @@ -711,7 +552,7 @@ def _normalize_embedding(embedding): @dataclass -class _LlamaSamplingParams: +class LlamaSamplingParams: n_prev: int = 64 n_probs: int = 0 top_k: int = 40 @@ -738,8 +579,8 @@ class _LlamaSamplingParams: @dataclass -class _LlamaSamplingContext: - params: _LlamaSamplingParams = field(default_factory=_LlamaSamplingParams) +class LlamaSamplingContext: + params: LlamaSamplingParams = field(default_factory=LlamaSamplingParams) mirostat_mu: ctypes.c_float = field(default_factory=ctypes.c_float) grammar: Optional[LlamaGrammar] = None # NOTE: Missing parsed_grammar @@ -753,7 +594,7 @@ def reset(self): self.grammar.reset() def cp(self): - return _LlamaSamplingContext( + return LlamaSamplingContext( params=self.params, mirostat_mu=self.mirostat_mu, grammar=self.grammar, @@ -767,12 +608,12 @@ def last(self) -> Optional[int]: else: return None - def prev_str(self, ctx_main: _LlamaContext, n: int) -> str: + def prev_str(self, ctx_main: LlamaContext, n: int) -> str: return ctx_main.model.detokenize(self.prev[-n:]).decode("utf-8") def sample( self, - ctx_main: _LlamaContext, + ctx_main: LlamaContext, idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None, ): @@ -790,7 +631,7 @@ def sample( for token, logit_bias in self.params.logit_bias.items(): logits_array[token] += logit_bias - token_data_array = _LlamaTokenDataArray( + token_data_array = LlamaTokenDataArray( n_vocab=n_vocab ) # TODO: Only create this once token_data_array.copy_logits(logits_array) @@ -862,7 +703,141 @@ def sample( id = ctx_main.sample_token(token_data_array) return id - def accept(self, ctx_main: _LlamaContext, id: int, apply_grammar: bool): + def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): if apply_grammar and self.grammar is not None: ctx_main.grammar_accept_token(self.grammar, id) self.prev.append(id) + + +from typing import List, Callable, Optional, Union +import ctypes +import llama_cpp + +class CustomSampler: + def __init__(self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None]): + self.apply_func = apply_func + + def apply_wrapper(sampler: llama_cpp.llama_sampler_p, cur_p: llama_cpp.llama_token_data_array_p): + self.apply_func(cur_p) + + def free_wrapper(sampler: llama_cpp.llama_sampler_p): + pass + + sampler_i = llama_cpp.llama_sampler_i() + sampler_i.apply = llama_cpp.llama_sampler_i_apply(apply_wrapper) + self._apply_wrapper_ref = apply_wrapper + + sampler_i.name = llama_cpp.llama_sampler_i_name(0) + sampler_i.accept = llama_cpp.llama_sampler_i_accept(0) + sampler_i.reset = llama_cpp.llama_sampler_i_reset(0) + sampler_i.clone = llama_cpp.llama_sampler_i_clone(0) + sampler_i.free = llama_cpp.llama_sampler_i_free(0) + + self.sampler = llama_cpp.llama_sampler() + self.sampler.iface = ctypes.pointer(sampler_i) + self.sampler.ctx = None + + def get_sampler(self) -> llama_cpp.llama_sampler_p: + return ctypes.pointer(self.sampler) + +class LlamaSampler: + def __init__(self): + params = llama_cpp.llama_sampler_chain_params() + self.sampler = llama_cpp.llama_sampler_chain_init(params) + self.samplers: List[llama_cpp.llama_sampler_p] = [] + self.custom_samplers: List[Tuple[int, CustomSampler]] = [] + + def add_greedy(self): + sampler = llama_cpp.llama_sampler_init_greedy() + self._add_sampler(sampler) + + def add_dist(self, seed: int): + sampler = llama_cpp.llama_sampler_init_dist(seed) + self._add_sampler(sampler) + + def add_softmax(self): + sampler = llama_cpp.llama_sampler_init_softmax() + self._add_sampler(sampler) + + def add_top_k(self, k: int): + sampler = llama_cpp.llama_sampler_init_top_k(k) + self._add_sampler(sampler) + + def add_top_p(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_top_p(p, min_keep) + self._add_sampler(sampler) + + def add_min_p(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_min_p(p, min_keep) + self._add_sampler(sampler) + + def add_tail_free(self, z: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_tail_free(z, min_keep) + self._add_sampler(sampler) + + def add_typical(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_typical(p, min_keep) + self._add_sampler(sampler) + + def add_temp(self, temp: float): + sampler = llama_cpp.llama_sampler_init_temp(temp) + self._add_sampler(sampler) + + def add_temp_ext(self, t: float, delta: float, exponent: float): + sampler = llama_cpp.llama_sampler_init_temp_ext(t, delta, exponent) + self._add_sampler(sampler) + + def add_mirostat(self, n_vocab: int, seed: int, tau: float, eta: float, m: int): + sampler = llama_cpp.llama_sampler_init_mirostat( + n_vocab, seed, tau, eta, m + ) + self._add_sampler(sampler) + + def add_mirostat_v2(self, seed: int, tau: float, eta: float): + sampler = llama_cpp.llama_sampler_init_mirostat_v2(seed, tau, eta) + self._add_sampler(sampler) + + def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): + sampler = llama_cpp.llama_sampler_init_grammar(model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8")) + self._add_sampler(sampler) + + def add_penalties(self, n_vocab: int, special_eos_id: int, linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, penalize_nl: bool, ignore_eos: bool): + sampler = llama_cpp.llama_sampler_init_penalties(n_vocab, special_eos_id, linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, penalize_nl, ignore_eos) + self._add_sampler(sampler) + + def init_logit_bias(self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p): + sampler = llama_cpp.llama_sampler_init_logit_bias(n_vocab, n_logit_bias, logit_bias) + self._add_sampler(sampler) + + def add_custom(self, apply_func: Callable[[llama_cpp.llama_token_data_array], None]): + custom_sampler = CustomSampler(apply_func) + sampler = custom_sampler.get_sampler() + self._add_sampler(sampler) + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + self.custom_samplers.append((llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler)) + + def _add_sampler(self, sampler: llama_cpp.llama_sampler_p): + assert self.sampler is not None + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + self.samplers.append(sampler) + + def get_seed(self) -> int: + assert self.sampler is not None + return llama_cpp.llama_sampler_get_seed(self.sampler) + + def sample(self, ctx: LlamaContext, idx: int) -> int: + assert self.sampler is not None + return llama_cpp.llama_sampler_sample(self.sampler, ctx.ctx, idx) + + def close(self): + if self.sampler: + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + for i, _ in reversed(self.custom_samplers): + llama_cpp.llama_sampler_chain_remove(self.sampler, i) + llama_cpp.llama_sampler_free(self.sampler) + self.sampler = None + self.samplers.clear() + self.custom_samplers.clear() + + def __del__(self): + self.close() diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 7638170a9..157af692f 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -5,21 +5,24 @@ import llama_cpp # enum ggml_log_level { -# GGML_LOG_LEVEL_ERROR = 2, -# GGML_LOG_LEVEL_WARN = 3, -# GGML_LOG_LEVEL_INFO = 4, -# GGML_LOG_LEVEL_DEBUG = 5 +# GGML_LOG_LEVEL_NONE = 0, +# GGML_LOG_LEVEL_INFO = 1, +# GGML_LOG_LEVEL_WARN = 2, +# GGML_LOG_LEVEL_ERROR = 3, +# GGML_LOG_LEVEL_DEBUG = 4, # }; GGML_LOG_LEVEL_TO_LOGGING_LEVEL = { - 2: logging.ERROR, - 3: logging.WARNING, - 4: logging.INFO, - 5: logging.DEBUG, + 0: logging.CRITICAL, + 1: logging.INFO, + 2: logging.WARNING, + 3: logging.ERROR, + 4: logging.DEBUG, } logger = logging.getLogger("llama-cpp-python") +# typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); @llama_cpp.llama_log_callback def llama_log_callback( level: int, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7fac936d1..557134a6e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -46,15 +46,7 @@ import numpy as np import numpy.typing as npt -from ._internals import ( - _LlamaModel, # type: ignore - _LlamaContext, # type: ignore - _LlamaBatch, # type: ignore - _LlamaTokenDataArray, # type: ignore - _LlamaSamplingParams, # type: ignore - _LlamaSamplingContext, # type: ignore - _normalize_embedding, # type: ignore -) +import llama_cpp._internals as internals from ._logger import set_verbose from ._utils import suppress_stdout_stderr @@ -368,7 +360,7 @@ def __init__( self._model = self._stack.enter_context( contextlib.closing( - _LlamaModel( + internals.LlamaModel( path_model=self.model_path, params=self.model_params, verbose=self.verbose, @@ -388,7 +380,7 @@ def __init__( self._ctx = self._stack.enter_context( contextlib.closing( - _LlamaContext( + internals.LlamaContext( model=self._model, params=self.context_params, verbose=self.verbose, @@ -398,7 +390,7 @@ def __init__( self._batch = self._stack.enter_context( contextlib.closing( - _LlamaBatch( + internals.LlamaBatch( n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx, @@ -410,7 +402,6 @@ def __init__( self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None if self.lora_path: - assert self._model.model is not None self._lora_adapter = llama_cpp.llama_lora_adapter_init( self._model.model, self.lora_path.encode("utf-8"), @@ -428,7 +419,6 @@ def free_lora_adapter(): self._stack.callback(free_lora_adapter) - assert self._ctx.ctx is not None if llama_cpp.llama_lora_adapter_set( self._ctx.ctx, self._lora_adapter, self.lora_scale ): @@ -453,7 +443,7 @@ def free_lora_adapter(): self._token_nl = self.token_nl() self._token_eos = self.token_eos() - self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab) + self._candidates = internals.LlamaTokenDataArray(n_vocab=self._n_vocab) self.n_tokens = 0 self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) @@ -542,14 +532,14 @@ def free_lora_adapter(): f"Using fallback chat format: {self.chat_format}", file=sys.stderr ) + self._sampler = None + @property def ctx(self) -> llama_cpp.llama_context_p: - assert self._ctx.ctx is not None return self._ctx.ctx @property def model(self) -> llama_cpp.llama_model_p: - assert self._model.model is not None return self._model.model @property @@ -618,8 +608,8 @@ def set_seed(self, seed: int): Args: seed: The random seed. """ - assert self._ctx.ctx is not None - llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) + # TODO: Fix this + # llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) def reset(self): """Reset the model state.""" @@ -631,8 +621,6 @@ def eval(self, tokens: Sequence[int]): Args: tokens: The list of tokens to evaluate. """ - assert self._ctx.ctx is not None - assert self._batch.batch is not None self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1) for i in range(0, len(tokens), self.n_batch): batch = tokens[i : min(len(tokens), i + self.n_batch)] @@ -662,6 +650,93 @@ def eval(self, tokens: Sequence[int]): # Update n_tokens self.n_tokens += n_tokens + def _init_sampler( + self, + top_k: int = 40, + top_p: float = 0.95, + min_p: float = 0.05, + typical_p: float = 1.0, + temp: float = 0.80, + repeat_penalty: float = 1.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + tfs_z: float = 1.0, + mirostat_mode: int = 0, + mirostat_eta: float = 0.1, + mirostat_tau: float = 5.0, + penalize_nl: bool = True, + logits_processor: Optional[LogitsProcessorList] = None, + grammar: Optional[LlamaGrammar] = None, + seed: Optional[int] = None, + ): + sampler = internals.LlamaSampler() + + if logits_processor is not None: + # Create and add a custom sampler + def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): + size = token_data_array.contents.size + data_soa = token_data_array.contents.data + data_soa_address = ctypes.addressof(data_soa.contents) + # NOTE: This is probably broken + recarray = np.recarray( + shape=(size,), + dtype=np.dtype( + [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True + ), + buf=(llama_cpp.llama_token_data * size).from_address(data_soa_address), + ) + for logit_processor in logits_processor: + recarray.logit[:] = logit_processor(self._input_ids, recarray.logit) + sampler.add_custom(apply_func) + + sampler.add_penalties( + n_vocab=self._n_vocab, + special_eos_id=self._token_eos, + linefeed_id=self._token_nl, + penalty_last_n=self.last_n_tokens_size, + penalty_repeat=repeat_penalty, + penalty_freq=frequency_penalty, + penalty_present=presence_penalty, + penalize_nl=penalize_nl, + ignore_eos=False + ) + + if grammar is not None: + sampler.add_grammar(self._model, grammar) + + if temp < 0.0: + sampler.add_softmax() + sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + elif temp == 0.0: + sampler.add_greedy() + else: + if mirostat_mode == 1: + mirostat_m = 100 + sampler.add_mirostat( + self._n_vocab, + seed or llama_cpp.LLAMA_DEFAULT_SEED, + mirostat_tau, + mirostat_eta, + mirostat_m, + ) + elif mirostat_mode == 2: + sampler.add_mirostat_v2( + seed or llama_cpp.LLAMA_DEFAULT_SEED, + mirostat_tau, + mirostat_eta, + ) + else: + n_probs = 0 + min_keep = max(1, n_probs) + sampler.add_top_k(top_k) + sampler.add_tail_free(tfs_z, min_keep) + sampler.add_typical(typical_p, min_keep) + sampler.add_top_p(top_p, min_keep) + sampler.add_min_p(min_p, min_keep) + sampler.add_temp(temp) + sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + return sampler + def sample( self, top_k: int = 40, @@ -692,49 +767,35 @@ def sample( Returns: The sampled token. """ - assert self._ctx is not None assert self.n_tokens > 0 - if idx is None: - logits: npt.NDArray[np.single] = self._scores[-1, :] - else: - logits = self._scores[idx, :] - - if logits_processor is not None: - logits[:] = ( - logits_processor(self._input_ids, logits) - if idx is None - else logits_processor(self._input_ids[: idx + 1], logits) + tmp_sampler = False + + if self._sampler is None: + tmp_sampler = True + self._sampler = self._init_sampler( + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, ) - sampling_params = _LlamaSamplingParams( - top_k=top_k, - top_p=top_p, - min_p=min_p, - tfs_z=tfs_z, - typical_p=typical_p, - temp=temp, - penalty_last_n=self.last_n_tokens_size, - penalty_repeat=repeat_penalty, - penalty_freq=frequency_penalty, - penalty_present=presence_penalty, - mirostat=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - penalize_nl=penalize_nl, - ) - sampling_context = _LlamaSamplingContext( - params=sampling_params, - grammar=grammar, - ) - sampling_context.prev = list(self.eval_tokens) - id = sampling_context.sample(ctx_main=self._ctx, logits_array=logits) - sampling_context.accept( - ctx_main=self._ctx, - id=id, - apply_grammar=grammar is not None, - ) - return id + assert self.ctx is not None + token = self._sampler.sample(self._ctx, -1) + if tmp_sampler: + self._sampler = None + return token def generate( self, @@ -756,6 +817,7 @@ def generate( logits_processor: Optional[LogitsProcessorList] = None, stopping_criteria: Optional[StoppingCriteriaList] = None, grammar: Optional[LlamaGrammar] = None, + seed: Optional[int] = None, ) -> Generator[int, Optional[Sequence[int]], None]: """Create a generator of tokens from a prompt. @@ -778,6 +840,24 @@ def generate( """ # Reset mirostat sampling self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau) + self._sampler = self._init_sampler( + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, + seed=seed, + ) # Check for kv cache prefix match if reset and self.n_tokens > 0: @@ -799,9 +879,9 @@ def generate( if reset: self.reset() - # Reset the grammar - if grammar is not None: - grammar.reset() + # # Reset the grammar + # if grammar is not None: + # grammar.reset() sample_idx = self.n_tokens + len(tokens) - 1 tokens = list(tokens) @@ -867,7 +947,6 @@ def create_embedding( Returns: An embedding object. """ - assert self._model.model is not None model_name: str = model if model is not None else self.model_path input = input if isinstance(input, list) else [input] @@ -912,7 +991,6 @@ def embed( Returns: A list of embeddings """ - assert self._ctx.ctx is not None n_embd = self.n_embd() n_batch = self.n_batch @@ -926,7 +1004,7 @@ def embed( ) if self.verbose: - llama_cpp.llama_reset_timings(self._ctx.ctx) + llama_cpp.llama_perf_context_reset(self._ctx.ctx) if isinstance(input, str): inputs = [input] @@ -940,7 +1018,6 @@ def embed( data: Union[List[List[float]], List[List[List[float]]]] = [] def decode_batch(seq_sizes: List[int]): - assert self._ctx.ctx is not None llama_cpp.llama_kv_cache_clear(self._ctx.ctx) self._ctx.decode(self._batch) self._batch.reset() @@ -955,7 +1032,7 @@ def decode_batch(seq_sizes: List[int]): for j in range(size) ] if normalize: - embedding = [_normalize_embedding(e) for e in embedding] + embedding = [internals.normalize_embedding(e) for e in embedding] data.append(embedding) pos += size else: @@ -963,7 +1040,7 @@ def decode_batch(seq_sizes: List[int]): ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i) embedding: List[float] = ptr[:n_embd] if normalize: - embedding = _normalize_embedding(embedding) + embedding = internals.normalize_embedding(embedding) data.append(embedding) # init state @@ -1006,7 +1083,7 @@ def decode_batch(seq_sizes: List[int]): decode_batch(s_batch) if self.verbose: - llama_cpp.llama_print_timings(self._ctx.ctx) + llama_cpp.llama_perf_context_print(self._ctx.ctx) output = data[0] if isinstance(input, str) else data @@ -1048,7 +1125,6 @@ def _create_completion( ) -> Union[ Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: - assert self._ctx is not None assert suffix is None or suffix.__class__ is str completion_id: str = f"cmpl-{str(uuid.uuid4())}" @@ -1211,8 +1287,9 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache miss", file=sys.stderr) - if seed is not None: - self._ctx.set_rng_seed(seed) + # TODO: Fix this + # if seed is not None: + # self._ctx.set_rng_seed(seed) finish_reason = "length" multibyte_fix = 0 @@ -1233,8 +1310,8 @@ def logit_bias_processor( stopping_criteria=stopping_criteria, logits_processor=logits_processor, grammar=grammar, + seed=seed, ): - assert self._model.model is not None if llama_cpp.llama_token_is_eog(self._model.model, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" @@ -2019,7 +2096,6 @@ def __setstate__(self, state): self.__init__(**state) def save_state(self) -> LlamaState: - assert self._ctx.ctx is not None if self.verbose: print("Llama.save_state: saving llama state", file=sys.stderr) state_size = llama_cpp.llama_get_state_size(self._ctx.ctx) @@ -2049,7 +2125,6 @@ def save_state(self) -> LlamaState: ) def load_state(self, state: LlamaState) -> None: - assert self._ctx.ctx is not None # Only filling in up to `n_tokens` and then zero-ing out the rest self.scores[: state.n_tokens, :] = state.scores.copy() self.scores[state.n_tokens :, :] = 0.0 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ecaf2369c..261d7c6b4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -233,6 +233,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_DEFAULT_SEED 0xFFFFFFFF LLAMA_DEFAULT_SEED = 0xFFFFFFFF +# define LLAMA_TOKEN_NULL -1 +LLAMA_TOKEN_NULL = -1 + # define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' LLAMA_FILE_MAGIC_GGLA = 0x67676C61 @@ -244,8 +247,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 8 -LLAMA_SESSION_VERSION = 8 +# define LLAMA_SESSION_VERSION 9 +LLAMA_SESSION_VERSION = 9 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ @@ -260,6 +263,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa llama_context_p = NewType("llama_context_p", int) llama_context_p_ctypes = ctypes.c_void_p +# # struct llama_sampler; +# llama_sampler_p = NewType("llama_sampler_p", int) +# llama_sampler_p_ctypes = ctypes.c_void_p # typedef int32_t llama_pos; llama_pos = ctypes.c_int32 @@ -561,6 +567,7 @@ class llama_token_data(ctypes.Structure): # typedef struct llama_token_data_array { # llama_token_data * data; # size_t size; +# int64_t selected; // this is the index in the data array (i.e. not the token id) # bool sorted; # } llama_token_data_array; class llama_token_data_array(ctypes.Structure): @@ -569,16 +576,19 @@ class llama_token_data_array(ctypes.Structure): Attributes: data (ctypes.Array[llama_token_data]): token data size (int): size of the array + selected (int): index in the data array (i.e. not the token id) sorted (bool): whether the array is sorted""" if TYPE_CHECKING: data: CtypesArray[llama_token_data] size: int + selected: int sorted: bool _fields_ = [ ("data", llama_token_data_p), ("size", ctypes.c_size_t), + ("selected", ctypes.c_int64), ("sorted", ctypes.c_bool), ] @@ -797,7 +807,6 @@ class llama_model_params(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggerganov/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t seed; // RNG seed, -1 for random # uint32_t n_ctx; // text context, 0 = from model # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size @@ -830,6 +839,7 @@ class llama_model_params(ctypes.Structure): # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU # bool flash_attn; // whether to use flash attention [EXPERIMENTAL] +# bool no_perf; // whether to measure performance timings # // Abort callback @@ -842,7 +852,6 @@ class llama_context_params(ctypes.Structure): """Parameters for llama_context Attributes: - seed (int): RNG seed, -1 for random n_ctx (int): text context, 0 = from model n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size @@ -873,7 +882,6 @@ class llama_context_params(ctypes.Structure): """ if TYPE_CHECKING: - seed: int n_ctx: int n_batch: int n_ubatch: int @@ -903,7 +911,6 @@ class llama_context_params(ctypes.Structure): abort_callback_data: ctypes.c_void_p _fields_ = [ - ("seed", ctypes.c_uint32), ("n_ctx", ctypes.c_uint32), ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), @@ -1009,101 +1016,44 @@ class llama_model_quantize_params(ctypes.Structure): ] -# // grammar types -# struct llama_grammar; -llama_grammar_p = ctypes.c_void_p - -# // grammar element type -# enum llama_gretype { -# // end of rule definition -# LLAMA_GRETYPE_END = 0, - -# // start of alternate definition for rule -# LLAMA_GRETYPE_ALT = 1, - -# // non-terminal element: reference to rule -# LLAMA_GRETYPE_RULE_REF = 2, - -# // terminal element: character (code point) -# LLAMA_GRETYPE_CHAR = 3, - -# // inverse char(s) ([^a], [^a-b] [^abc]) -# LLAMA_GRETYPE_CHAR_NOT = 4, - -# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to -# // be an inclusive range ([a-z]) -# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, +# typedef struct llama_logit_bias { +# llama_token token; +# float bias; +# } llama_logit_bias; +class llama_logit_bias(ctypes.Structure): + """Used to store logit bias -# // modifies a preceding LLAMA_GRETYPE_CHAR or -# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) -# LLAMA_GRETYPE_CHAR_ALT = 6, + Attributes: + token (llama_token): token id + bias (float): bias""" -# // any character (.) -# LLAMA_GRETYPE_CHAR_ANY = 7, -# }; -LLAMA_GRETYPE_END = 0 -LLAMA_GRETYPE_ALT = 1 -LLAMA_GRETYPE_RULE_REF = 2 -LLAMA_GRETYPE_CHAR = 3 -LLAMA_GRETYPE_CHAR_NOT = 4 -LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 -LLAMA_GRETYPE_CHAR_ALT = 6 -LLAMA_GRETYPE_CHAR_ANY = 7 - - -# typedef struct llama_grammar_element { -# enum llama_gretype type; -# uint32_t value; // Unicode code point or rule ID -# } llama_grammar_element; -class llama_grammar_element(ctypes.Structure): if TYPE_CHECKING: - type: int - value: int + token: llama_token + bias: float _fields_ = [ - ("type", ctypes.c_int), - ("value", ctypes.c_uint32), + ("token", llama_token), + ("bias", ctypes.c_float), ] -llama_grammar_element_p = ctypes.POINTER(llama_grammar_element) +llama_logit_bias_p = ctypes.POINTER(llama_logit_bias) -# // performance timing information -# struct llama_timings { -# double t_start_ms; -# double t_end_ms; -# double t_load_ms; -# double t_sample_ms; -# double t_p_eval_ms; -# double t_eval_ms; +# typedef struct llama_sampler_chain_params { +# bool no_perf; // whether to measure performance timings +# } llama_sampler_chain_params; +class llama_sampler_chain_params(ctypes.Structure): + """Parameters for llama_sampler_chain + + Attributes: + no_perf (bool): whether to measure performance timings""" -# int32_t n_sample; -# int32_t n_p_eval; -# int32_t n_eval; -# }; -class llama_timings(ctypes.Structure): if TYPE_CHECKING: - t_start_ms: float - t_end_ms: float - t_load_ms: float - t_sample_ms: float - t_p_eval_ms: float - t_eval_ms: float - n_sample: int - n_p_eval: int - n_eval: int + no_perf: bool _fields_ = [ - ("t_start_ms", ctypes.c_double), - ("t_end_ms", ctypes.c_double), - ("t_load_ms", ctypes.c_double), - ("t_sample_ms", ctypes.c_double), - ("t_p_eval_ms", ctypes.c_double), - ("t_eval_ms", ctypes.c_double), - ("n_sample", ctypes.c_int32), - ("n_p_eval", ctypes.c_int32), - ("n_eval", ctypes.c_int32), + ("no_perf", ctypes.c_bool), ] @@ -1126,7 +1076,7 @@ class llama_chat_message(ctypes.Structure): # // Helpers for getting default parameters -# LLAMA_API struct llama_model_params llama_model_default_params(void); +# LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( "llama_model_default_params", [], @@ -1137,7 +1087,7 @@ def llama_model_default_params() -> llama_model_params: ... -# LLAMA_API struct llama_context_params llama_context_default_params(void); +# LLAMA_API struct llama_context_params llama_context_default_params(void); @ctypes_function( "llama_context_default_params", [], @@ -1148,6 +1098,17 @@ def llama_context_default_params() -> llama_context_params: ... +# LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); +@ctypes_function( + "llama_sampler_chain_default_params", + [], + llama_sampler_chain_params, +) +def llama_sampler_chain_default_params() -> llama_sampler_chain_params: + """Get default parameters for llama_sampler_chain""" + ... + + # LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); @ctypes_function( "llama_model_quantize_default_params", @@ -1200,8 +1161,7 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): - ... +def llama_numa_init(numa: int, /): ... # // Optional: an auto threadpool gets created in ggml if not passed explicitly @@ -1228,7 +1188,7 @@ def llama_backend_free(): # LLAMA_API struct llama_model * llama_load_model_from_file( # const char * path_model, -# struct llama_model_params params); +# struct llama_model_params params); @ctypes_function( "llama_load_model_from_file", [ctypes.c_char_p, llama_model_params], @@ -1236,8 +1196,7 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: - ... +) -> Optional[llama_model_p]: ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1246,8 +1205,7 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): - ... +def llama_free_model(model: llama_model_p, /): ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1260,8 +1218,7 @@ def llama_free_model(model: llama_model_p, /): ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # // Frees all allocated memory @@ -1282,104 +1239,87 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: - ... +def llama_time_us() -> int: ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: - ... +def llama_max_devices() -> int: ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: - ... +def llama_supports_mmap() -> bool: ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: - ... +def llama_supports_mlock() -> bool: ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: - ... - - -# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); -@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: - ... +def llama_supports_gpu_offload() -> bool: ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: - ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: - ... +def llama_n_batch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: - ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: - ... - - -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); -@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: - ... - - -# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); -@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: - ... - - -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); -@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: - ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... +def llama_n_vocab(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: - ... +def llama_n_embd(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: - ... +def llama_n_layer(model: llama_model_p, /) -> int: ... + + +# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); +@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... + + +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) +def llama_pooling_type(ctx: llama_context_p, /) -> int: ... + + +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); +@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_vocab_type(model: llama_model_p, /) -> int: ... + + +# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); +@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_rope_type(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor @@ -2040,7 +1980,7 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // Returns the *actual* size in bytes of the state -# // (rng, logits, embedding and kv_cache) +# // (logits, embedding and kv_cache) # // Only use when saving the state, not when restoring it, otherwise the size may be too small. # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) @@ -2170,8 +2110,7 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2199,8 +2138,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # LLAMA_API bool llama_state_save_file( @@ -2224,8 +2162,7 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2250,8 +2187,7 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2349,8 +2285,7 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2380,8 +2315,7 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # // @@ -2686,8 +2620,7 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2696,8 +2629,7 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2706,8 +2638,7 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2772,14 +2703,12 @@ def llama_token_nl(model: llama_model_p, /) -> int: # LLAMA_API bool llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: - ... +def llama_add_bos_token(model: llama_model_p, /) -> bool: ... # LLAMA_API bool llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: - ... +def llama_add_eos_token(model: llama_model_p, /) -> bool: ... # // Codellama infill tokens @@ -2792,20 +2721,17 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: - ... +def llama_token_middle(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: - ... +def llama_token_suffix(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: - ... +def llama_token_eot(model: llama_model_p, /) -> int: ... # // @@ -3006,419 +2932,296 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: - ... +) -> int: ... # // -# // Grammar +# // Sampling API +# // +# // Sample usage: +# // +# // // prepare the sampling chain at the start +# // auto sparams = llama_sampler_chain_default_params(); +# // +# // llama_sampler * smpl = llama_sampler_chain_init(sparams); +# // +# // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); +# // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); +# // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); +# // +# // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" +# // // this sampler will be responsible to select the actual token +# // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); +# // +# // ... +# // +# // // decoding loop: +# // while (...) { +# // ... +# // +# // llama_decode(ctx, batch); +# // +# // // sample from the logits of the last token in the batch +# // const llama_token id = llama_sampler_sample(smpl, ctx, -1); +# // +# // // accepting the token updates the internal state of certain samplers (e.g. grammar, repetition, etc.) +# // llama_sampler_accept(smpl, id); +# // ... +# // } +# // +# // llama_sampler_free(smpl); +# // +# // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU). +# // TODO: in the future, the entire sampling API that uses llama_model should start using llama_vocab # // +# typedef void * llama_sampler_context_t; +llama_sampler_context_t = ctypes.c_void_p -# LLAMA_API struct llama_grammar * llama_grammar_init( -# const llama_grammar_element ** rules, -# size_t n_rules, -# size_t start_rule_index); -@ctypes_function( - "llama_grammar_init", - [ - ctypes.POINTER(llama_grammar_element_p), - ctypes.c_size_t, - ctypes.c_size_t, - ], - llama_grammar_p, -) -def llama_grammar_init( - rules: CtypesArray[ - CtypesPointer[llama_grammar_element] - ], # NOTE: This might be wrong type sig - n_rules: Union[ctypes.c_size_t, int], - start_rule_index: Union[ctypes.c_size_t, int], - /, -) -> Optional[llama_grammar_p]: - """Initialize a grammar from a set of rules.""" - ... + +# // user code can implement the interface below in order to create custom llama_sampler +# struct llama_sampler_i { +# const char * (*name) (const struct llama_sampler * smpl); // can be NULL +# void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL +# void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required +# void (*reset) ( struct llama_sampler * smpl); // can be NULL +# struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL +# void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL +# +# // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph +# //void (*apply_ggml) (struct llama_sampler * smpl, ...); +# }; +class llama_sampler_i(ctypes.Structure): ... -# LLAMA_API void llama_grammar_free(struct llama_grammar * grammar); -@ctypes_function( - "llama_grammar_free", - [llama_grammar_p], - None, +# struct llama_sampler { +# struct llama_sampler_i * iface; +# llama_sampler_context_t ctx; +# }; +class llama_sampler(ctypes.Structure): + _fields_ = [ + ("iface", ctypes.POINTER(llama_sampler_i)), + ("ctx", llama_sampler_context_t), + ] + + +if TYPE_CHECKING: + llama_sampler_p = CtypesPointer[llama_sampler] + +llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler) + +llama_sampler_i_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes) +llama_sampler_i_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token) +llama_sampler_i_apply = ctypes.CFUNCTYPE( + None, llama_sampler_p_ctypes, llama_token_data_array_p ) -def llama_grammar_free(grammar: llama_grammar_p, /): - """Free a grammar.""" - ... +llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) +llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i._fields_ = [ + ("name", llama_sampler_i_name), + ("accept", llama_sampler_i_accept), + ("apply", llama_sampler_i_apply), + ("reset", llama_sampler_i_reset), + ("clone", llama_sampler_i_clone), + ("free", llama_sampler_i_free), +] -# LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar); + +# // mirror of llama_sampler_i: +# LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); @ctypes_function( - "llama_grammar_copy", - [llama_grammar_p], - llama_grammar_p, + "llama_sampler_name", + [llama_sampler_p_ctypes], + ctypes.c_char_p, ) -def llama_grammar_copy(grammar: llama_grammar_p, /) -> llama_grammar_p: - """Copy a grammar.""" - ... +def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... -# /// @details Apply constraints from grammar -# LLAMA_API void llama_grammar_sample( -# const struct llama_grammar * grammar, -# const struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); @ctypes_function( - "llama_grammar_sample", - [ - llama_grammar_p, - llama_context_p_ctypes, - llama_token_data_array_p, - ], + "llama_sampler_accept", + [llama_sampler_p_ctypes, llama_token], None, ) -def llama_grammar_sample( - grammar: llama_grammar_p, - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -): - """Apply constraints from grammar""" - ... +def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ... -# LLAMA_API DEPRECATED(void llama_sample_grammar( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const struct llama_grammar * grammar), -# "use llama_grammar_sample instead"); +# LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); @ctypes_function( - "llama_sample_grammar", - [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], + "llama_sampler_apply", + [llama_sampler_p_ctypes, llama_token_data_array_p], None, ) -def llama_sample_grammar( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - grammar, # type: llama_grammar_p - /, -): - """Apply constraints from grammar - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - grammar: A grammar object containing the rules and constraints to apply to the generated text. - """ - ... +def llama_sampler_apply( + smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], / +): ... -# /// @details Accepts the sampled token into the grammar -# LLAMA_API void llama_grammar_accept_token( -# struct llama_grammar * grammar, -# struct llama_context * ctx, -# llama_token token); +# LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); @ctypes_function( - "llama_grammar_accept_token", - [llama_grammar_p, llama_context_p_ctypes, llama_token], + "llama_sampler_reset", + [llama_sampler_p_ctypes], None, ) -def llama_grammar_accept_token( - grammar: llama_grammar_p, - ctx: llama_context_p, - token: Union[llama_token, int], - /, -): - """Accepts the sampled token into the grammar""" - ... +def llama_sampler_reset(smpl: llama_sampler_p, /): ... -# // -# // Sampling functions -# // +# LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); +@ctypes_function( + "llama_sampler_clone", + [llama_sampler_p_ctypes], + llama_sampler_p_ctypes, +) +def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... -# // Sets the current rng seed. -# LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed); +# // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) +# LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( - "llama_set_rng_seed", - [llama_context_p_ctypes, ctypes.c_uint32], + "llama_sampler_free", + [llama_sampler_p_ctypes], None, ) -def llama_set_rng_seed(ctx: llama_context_p, seed: Union[ctypes.c_uint32, int], /): - """Sets the current rng seed.""" - ... +def llama_sampler_free(smpl: llama_sampler_p, /): ... -# /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. -# /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. -# LLAMA_API void llama_sample_repetition_penalties( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const llama_token * last_tokens, -# size_t penalty_last_n, -# float penalty_repeat, -# float penalty_freq, -# float penalty_present); -@ctypes_function( - "llama_sample_repetition_penalties", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - llama_token_p, - ctypes.c_size_t, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ], - None, +# // llama_sampler_chain +# // a type of llama_sampler that can chain multiple samplers one after another +# +# LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); +@ctypes_function( + "llama_sampler_chain_init", + [llama_sampler_chain_params], + llama_sampler_p_ctypes, ) -def llama_sample_repetition_penalties( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - last_tokens_data: CtypesArray[llama_token], - penalty_last_n: Union[ctypes.c_size_t, int], - penalty_repeat: Union[ctypes.c_float, float], - penalty_freq: Union[ctypes.c_float, float], - penalty_present: Union[ctypes.c_float, float], - /, -): - """Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. - Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. - """ - ... +def llama_sampler_chain_init( + params: llama_sampler_chain_params, / +) -> llama_sampler_p: ... -# /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 -# /// @param logits Logits extracted from the original generation context. -# /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. -# /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. -# LLAMA_API void llama_sample_apply_guidance( -# struct llama_context * ctx, -# float * logits, -# float * logits_guidance, -# float scale); +# // important: takes ownership of the sampler object and will free it when llama_sampler_free is called +# LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); @ctypes_function( - "llama_sample_apply_guidance", - [ - llama_context_p_ctypes, - ctypes.POINTER(ctypes.c_float), - ctypes.POINTER(ctypes.c_float), - ctypes.c_float, - ], + "llama_sampler_chain_add", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], None, ) -def llama_sample_apply_guidance( - ctx: llama_context_p, - logits: CtypesArray[ctypes.c_float], - logits_guidance: CtypesArray[ctypes.c_float], - scale: Union[ctypes.c_float, float], - /, -): - """Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806""" - ... +def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ... -# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. -# LLAMA_API void llama_sample_softmax( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i); @ctypes_function( - "llama_sample_softmax", - [llama_context_p_ctypes, llama_token_data_array_p], - None, + "llama_sampler_chain_get", + [llama_sampler_p_ctypes, ctypes.c_int32], + llama_sampler_p_ctypes, ) -def llama_sample_softmax( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -): - """Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.""" - ... +def llama_sampler_chain_get( + chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / +) -> llama_sampler_p: ... -# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 -# LLAMA_API void llama_sample_top_k( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# int32_t k, -# size_t min_keep); +# LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); @ctypes_function( - "llama_sample_top_k", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_int32, ctypes.c_size_t], - None, + "llama_sampler_chain_n", + [llama_sampler_p_ctypes], + ctypes.c_int, ) -def llama_sample_top_k( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - k: Union[ctypes.c_int, int], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751""" - ... +def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... + + +# // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed +# LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); +@ctypes_function( + "llama_sampler_chain_remove", + [llama_sampler_p_ctypes, ctypes.c_int32], + llama_sampler_p_ctypes, +) +def llama_sampler_chain_remove( + chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / +) -> llama_sampler_p: ... + + +# // available samplers: +# +# LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); +@ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) +def llama_sampler_init_greedy() -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); +@ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) +def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... + + +# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. +# LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); +@ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) +def llama_sampler_init_softmax() -> llama_sampler_p: ... + + +# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +# LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); +@ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) +def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... # /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 -# LLAMA_API void llama_sample_top_p( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); @ctypes_function( - "llama_sample_top_p", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_top_p", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_top_p( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751""" - ... +def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 -# LLAMA_API void llama_sample_min_p( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); @ctypes_function( - "llama_sample_min_p", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_min_p", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_min_p( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841""" - ... +def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. -# LLAMA_API void llama_sample_tail_free( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float z, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_tail_free (float z, size_t min_keep); @ctypes_function( - "llama_sample_tail_free", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_tail_free", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_tail_free( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - z: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.""" - ... +def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: ... # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. -# LLAMA_API void llama_sample_typical( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); @ctypes_function( - "llama_sample_typical", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_typical", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_typical( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.""" - ... +def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... -# /// @details Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772. -# LLAMA_API void llama_sample_entropy( -# struct llama_context * ctx, -# llama_token_data_array * candidates_p, -# float min_temp, -# float max_temp, -# float exponent_val); -@ctypes_function( - "llama_sample_entropy", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ], - None, -) -def llama_sample_entropy( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - min_temp: Union[ctypes.c_float, float], - max_temp: Union[ctypes.c_float, float], - exponent_val: Union[ctypes.c_float, float], - /, -): - """Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.""" - ... +# LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); +@ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) +def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... -# LLAMA_API void llama_sample_temp( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float temp); +# /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. +# LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); @ctypes_function( - "llama_sample_temp", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float], - None, + "llama_sampler_init_temp_ext", + [ctypes.c_float, ctypes.c_float, ctypes.c_float], + llama_sampler_p_ctypes, ) -def llama_sample_temp( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - temp: Union[ctypes.c_float, float], - /, -): - """Temperature sampling described in academic paper "Generating Long Sequences with Sparse Transformers" https://arxiv.org/abs/1904.10509 - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - temp: The temperature value to use for the sampling. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - """ - ... +def llama_sampler_init_temp_ext( + t: float, delta: float, exponent: float +) -> llama_sampler_p: ... # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3427,46 +3230,20 @@ def llama_sample_temp( # /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. # /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. # /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. -# LLAMA_API llama_token llama_sample_token_mirostat( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float tau, -# float eta, -# int32_t m, -# float * mu); +# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( +# int32_t n_vocab, +# uint32_t seed, +# float tau, +# float eta, +# int32_t m); @ctypes_function( - "llama_sample_token_mirostat", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - ctypes.c_float, - ctypes.c_float, - ctypes.c_int32, - ctypes.POINTER(ctypes.c_float), - ], - llama_token, + "llama_sampler_init_mirostat", + [ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, ctypes.c_float, ctypes.c_int32], + llama_sampler_p_ctypes, ) -def llama_sample_token_mirostat( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - tau: Union[ctypes.c_float, float], - eta: Union[ctypes.c_float, float], - m: Union[ctypes.c_int, int], - mu: CtypesPointerOrRef[ctypes.c_float], - /, -) -> int: - """Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - m: The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. - mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - """ - ... +def llama_sampler_init_mirostat( + n_vocab: int, seed: int, tau: float, eta: float, m: int, / +) -> llama_sampler_p: ... # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3474,83 +3251,116 @@ def llama_sample_token_mirostat( # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. # /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. # /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. -# LLAMA_API llama_token llama_sample_token_mirostat_v2( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float tau, -# float eta, -# float * mu); -@ctypes_function( - "llama_sample_token_mirostat_v2", +# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( +# uint32_t seed, +# float tau, +# float eta); +@ctypes_function( + "llama_sampler_init_mirostat_v2", + [ctypes.c_uint32, ctypes.c_float, ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_mirostat_v2( + seed: int, tau: float, eta: float, / +) -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_grammar( +# const struct llama_model * model, +# const char * grammar_str, +# const char * grammar_root); +@ctypes_function( + "llama_sampler_init_grammar", + [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], + llama_sampler_p_ctypes, +) +def llama_sampler_init_grammar( + model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / +) -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, // llama_n_vocab() +# llama_token special_eos_id, // llama_token_eos() +# llama_token linefeed_id, // llama_token_nl() +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# float penalty_repeat, // 1.0 = disabled +# float penalty_freq, // 0.0 = disabled +# float penalty_present, // 0.0 = disabled +# bool penalize_nl, // consider newlines as a repeatable token +# bool ignore_eos); // ignore the end-of-sequence token +@ctypes_function( + "llama_sampler_init_penalties", [ - llama_context_p_ctypes, - llama_token_data_array_p, + ctypes.c_int32, + llama_token, + llama_token, + ctypes.c_int32, ctypes.c_float, ctypes.c_float, - ctypes.POINTER(ctypes.c_float), - ], - llama_token, -) -def llama_sample_token_mirostat_v2( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ctypes.c_float, + ctypes.c_bool, + ctypes.c_bool, ], - tau: Union[ctypes.c_float, float], - eta: Union[ctypes.c_float, float], - mu: CtypesPointerOrRef[ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_penalties( + n_vocab: int, + special_eos_id: int, + linefeed_id: int, + penalty_last_n: int, + penalty_repeat: float, + penalty_freq: float, + penalty_present: float, + penalize_nl: bool, + ignore_eos: bool, /, -) -> int: - """Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - """ - ... +) -> llama_sampler_p: ... -# /// @details Selects the token with the highest probability. -# /// Does not compute the token probabilities. Use llama_sample_softmax() instead. -# LLAMA_API llama_token llama_sample_token_greedy( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( +# int32_t n_vocab, +# int32_t n_logit_bias, +# const llama_logit_bias * logit_bias); @ctypes_function( - "llama_sample_token_greedy", - [llama_context_p_ctypes, llama_token_data_array_p], - llama_token, + "llama_sampler_init_logit_bias", + [ctypes.c_int32, ctypes.c_int32, llama_logit_bias_p], + llama_sampler_p_ctypes, ) -def llama_sample_token_greedy( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -) -> int: - """Selects the token with the highest probability.""" - ... +def llama_sampler_init_logit_bias( + n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], / +) -> llama_sampler_p: ... -# /// @details Randomly selects a token from the candidates based on their probabilities using the RNG of ctx. -# LLAMA_API llama_token llama_sample_token( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise +# LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); @ctypes_function( - "llama_sample_token", - [llama_context_p_ctypes, llama_token_data_array_p], + "llama_sampler_get_seed", + [llama_sampler_p_ctypes], + ctypes.c_uint32, +) +def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... + + +# /// @details Sample and accept a token from the idx-th output of the last evaluation +# // +# // Shorthand for: +# // const auto * logits = llama_get_logits_ith(ctx, idx); +# // llama_token_data_array cur_p = { ... init from logits ... }; +# // llama_sampler_apply(smpl, &cur_p); +# // auto token = cur_p.data[cur_p.selected].id; +# // llama_sampler_accept(smpl, token); +# // return token; +# // Returns the sampled token +# LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); +@ctypes_function( + "llama_sampler_sample", + [llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32], llama_token, ) -def llama_sample_token( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -) -> int: - """Randomly selects a token from the candidates based on their probabilities.""" - ... +def llama_sampler_sample( + smpl: llama_sampler_p, ctx: llama_context_p, idx: int, / +) -> int: ... # // @@ -3600,79 +3410,131 @@ def llama_split_prefix( ... -# Performance information +# // Print system information +# LLAMA_API const char * llama_print_system_info(void); +@ctypes_function("llama_print_system_info", [], ctypes.c_char_p) +def llama_print_system_info() -> bytes: ... -# LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); +# // Set callback for all future logging events. +# // If this is not called, or NULL is supplied, everything is output on stderr. +# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( - "llama_get_timings", - [llama_context_p_ctypes], - llama_timings, + "llama_log_set", + [ctypes.c_void_p, ctypes.c_void_p], + None, ) -def llama_get_timings(ctx: llama_context_p, /) -> llama_timings: - """Get performance information""" +def llama_log_set( + log_callback: Optional[CtypesFuncPointer], + user_data: ctypes.c_void_p, + /, +): + """Set callback for all future logging events. + + If this is not called, or NULL is supplied, everything is output on stderr.""" ... -# LLAMA_API void llama_print_timings(struct llama_context * ctx); +# // +# // Performance utils +# // +# // NOTE: Used by llama.cpp examples, avoid using in third-party apps. Instead, do your own performance measurements. +# // + + +# struct llama_perf_context_data { +# double t_start_ms; +# double t_load_ms; +# double t_p_eval_ms; +# double t_eval_ms; +# +# int32_t n_p_eval; +# int32_t n_eval; +# }; +class llama_perf_context_data(ctypes.Structure): + _fields_ = [ + ("t_start_ms", ctypes.c_double), + ("t_load_ms", ctypes.c_double), + ("t_p_eval_ms", ctypes.c_double), + ("t_eval_ms", ctypes.c_double), + ("n_p_eval", ctypes.c_int32), + ("n_eval", ctypes.c_int32), + ] + + +# struct llama_perf_sampler_data { +# double t_sample_ms; +# +# int32_t n_sample; +# }; +class llama_perf_sampler_data(ctypes.Structure): + _fields_ = [ + ("t_sample_ms", ctypes.c_double), + ("n_sample", ctypes.c_int32), + ] + + +# LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); @ctypes_function( - "llama_print_timings", + "llama_perf_context", + [llama_context_p_ctypes], + llama_perf_context_data, +) +def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... + + +# LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); +@ctypes_function( + "llama_perf_context_print", [llama_context_p_ctypes], None, ) -def llama_print_timings(ctx: llama_context_p, /): - """Print performance information""" - ... +def llama_perf_context_print(ctx: llama_context_p, /): ... -# LLAMA_API void llama_reset_timings(struct llama_context * ctx); +# LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); @ctypes_function( - "llama_reset_timings", + "llama_perf_context_reset", [llama_context_p_ctypes], None, ) -def llama_reset_timings(ctx: llama_context_p, /): - """Reset performance information""" - ... +def llama_perf_context_reset(ctx: llama_context_p, /): ... -# Print system information -# LLAMA_API const char * llama_print_system_info(void); +# // NOTE: the following work only with samplers constructed via llama_sampler_chain_init +# LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); @ctypes_function( - "llama_print_system_info", - [], - ctypes.c_char_p, + "llama_perf_sampler", + [llama_sampler_p_ctypes], + llama_perf_sampler_data, ) -def llama_print_system_info() -> bytes: - """Print system information""" - ... +def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ... -# NOTE: THIS IS CURRENTLY BROKEN AS ggml_log_callback IS NOT EXPOSED IN LLAMA.H -# // Set callback for all future logging events. -# // If this is not called, or NULL is supplied, everything is output on stderr. -# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); +# LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); @ctypes_function( - "llama_log_set", - [ctypes.c_void_p, ctypes.c_void_p], + "llama_perf_sampler_print", + [llama_sampler_p_ctypes], None, ) -def llama_log_set( - log_callback: Optional[CtypesFuncPointer], - user_data: ctypes.c_void_p, - /, -): - """Set callback for all future logging events. +def llama_perf_sampler_print(chain: llama_sampler_p, /): ... - If this is not called, or NULL is supplied, everything is output on stderr.""" - ... + +# LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); +@ctypes_function( + "llama_perf_sampler_reset", + [llama_sampler_p_ctypes], + None, +) +def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx); +# LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); @ctypes_function( - "llama_dump_timing_info_yaml", - [ctypes.c_void_p, llama_context_p_ctypes], + "llama_perf_dump_yaml", + [ctypes.POINTER(ctypes.c_void_p), llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): - ... +def llama_perf_dump_yaml( + stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / +): ... diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 4cd52c2d5..785aa88d7 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -2,11 +2,6 @@ # flake8: noqa from pathlib import Path -import sys -import ctypes -import enum -import typing -import dataclasses from itertools import groupby from typing import ( @@ -18,882 +13,16 @@ Union, ) -import llama_cpp.llama_cpp as llama_cpp - -class GrammarElementType(enum.IntEnum): - END = llama_cpp.LLAMA_GRETYPE_END - ALT = llama_cpp.LLAMA_GRETYPE_ALT - RULE_REF = llama_cpp.LLAMA_GRETYPE_RULE_REF - CHAR = llama_cpp.LLAMA_GRETYPE_CHAR - CHAR_NOT = llama_cpp.LLAMA_GRETYPE_CHAR_NOT - CHAR_RNG_UPPER = llama_cpp.LLAMA_GRETYPE_CHAR_RNG_UPPER - CHAR_ALT = llama_cpp.LLAMA_GRETYPE_CHAR_ALT - CHAR_ANY = llama_cpp.LLAMA_GRETYPE_CHAR_ANY - - -@dataclasses.dataclass -class GrammarElement: - type: GrammarElementType - value: int - - -@dataclasses.dataclass -class ParseState: - symbol_ids: typing.Dict[str, int] = dataclasses.field(default_factory=dict) - rules: typing.List[typing.List[GrammarElement]] = dataclasses.field(default_factory=list) - - -# static std::pair decode_utf8(const char * src) { -# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; -# uint8_t first_byte = static_cast(*src); -# uint8_t highbits = first_byte >> 4; -# int len = lookup[highbits]; -# uint8_t mask = (1 << (8 - len)) - 1; -# uint32_t value = first_byte & mask; -# const char * end = src + len; // may overrun! -# const char * pos = src + 1; -# for ( ; pos < end && *pos; pos++) { -# value = (value << 6) + (static_cast(*pos) & 0x3F); -# } -# return std::make_pair(value, pos); -# } -def decode_utf8(src: str) -> typing.Tuple[int, str]: - lookup: list[int] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4] - first_byte: int = ord(src[0]) - highbits: int = first_byte >> 4 - length: int = lookup[highbits] - mask: int = (1 << (8 - length)) - 1 - value: int = first_byte & mask - end: int = min(len(src), length) # Prevent overrun - - pos: int = 1 - for pos in range(1, end): - if not src[pos]: - break - value = (value << 6) + (ord(src[pos]) & 0x3F) - - return value, src[pos:] if pos < len(src) else "" - - -# static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { -# uint32_t next_id = static_cast(state.symbol_ids.size()); -# auto result = state.symbol_ids.emplace(std::string(src, len), next_id); -# return result.first->second; -# } -def get_symbol_id(state: ParseState, name: str) -> int: - next_id = len(state.symbol_ids) - return state.symbol_ids.setdefault(name, next_id) - - -# static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { -# uint32_t next_id = static_cast(state.symbol_ids.size()); -# state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id; -# return next_id; -# } -def generate_symbol_id(state: ParseState, base_name: str) -> int: - next_id = len(state.symbol_ids) - state.symbol_ids[f"{base_name}_{next_id}"] = next_id - return next_id - - -# static void add_rule( -# parse_state & state, -# uint32_t rule_id, -# const std::vector & rule) { -# if (state.rules.size() <= rule_id) { -# state.rules.resize(rule_id + 1); -# } -# state.rules[rule_id] = rule; -# } -def add_rule(state: ParseState, rule_id: int, rule: typing.List[GrammarElement]) -> None: - if len(state.rules) <= rule_id: - state.rules.extend([[]] * (rule_id + 1 - len(state.rules))) - state.rules[rule_id] = rule - - -# static bool is_digit_char(char c) { -# return '0' <= c && c <= '9'; -# } -def is_digit_char(c: str) -> bool: - return "0" <= c <= "9" - - -# static bool is_word_char(char c) { -# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c); -# } -def is_word_char(c: str) -> bool: - return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or is_digit_char(c) - - -# static std::pair parse_hex(const char * src, int size) { -# const char * pos = src; -# const char * end = src + size; -# uint32_t value = 0; -# for ( ; pos < end && *pos; pos++) { -# value <<= 4; -# char c = *pos; -# if ('a' <= c && c <= 'f') { -# value += c - 'a' + 10; -# } else if ('A' <= c && c <= 'F') { -# value += c - 'A' + 10; -# } else if ('0' <= c && c <= '9') { -# value += c - '0'; -# } else { -# break; -# } -# } -# if (pos != end) { -# throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src); -# } -# return std::make_pair(value, pos); -# } -def parse_hex(src: str, size: int) -> typing.Tuple[int, str]: - pos = 0 - value = 0 - for _ in range(size): - value <<= 4 - c = src[pos] - if "a" <= c <= "f": - value += ord(c) - ord("a") + 10 - elif "A" <= c <= "F": - value += ord(c) - ord("A") + 10 - elif "0" <= c <= "9": - value += ord(c) - ord("0") - else: - break - pos += 1 - if pos != size: - raise ValueError(f"expecting {size} hex chars at {src}") - return value, src[pos:] - - -# static const char * parse_space(const char * src, bool newline_ok) { -# const char * pos = src; -# while (*pos == ' ' || *pos == '\t' || *pos == '#' || -# (newline_ok && (*pos == '\r' || *pos == '\n'))) { -# if (*pos == '#') { -# while (*pos && *pos != '\r' && *pos != '\n') { -# pos++; -# } -# } else { -# pos++; -# } -# } -# return pos; -# } -def parse_space(src: str, newline_ok: bool) -> str: - pos = src - while pos and (pos[0] in (' ', '\t', '#') or (newline_ok and pos[0] in ('\r', '\n'))): - if pos[0] == "#": - while pos and pos[0] not in ("\r", "\n"): - pos = pos[1:] - else: - pos = pos[1:] - return pos - - -# static const char * parse_name(const char * src) { -# const char * pos = src; -# while (is_word_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting name at ") + src); -# } -# return pos; -# } -def parse_name(src: str) -> typing.Tuple[str, str]: - pos = src - while pos and is_word_char(pos[0]): - pos = pos[1:] - if pos == src: - raise ValueError(f"expecting name at {src}") - return src[:len(src) - len(pos)], pos - -# static const char * parse_int(const char * src) { -# const char * pos = src; -# while (is_digit_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting integer at ") + src); -# } -# return pos; -# } -def parse_int(src: str) -> typing.Tuple[int, str]: - pos = src - while pos and is_digit_char(pos[0]): - pos = pos[1:] - if pos == src: - raise ValueError(f"expecting integer at {src}") - return int(src[:len(src) - len(pos)]), pos - - -# static std::pair parse_char(const char * src) { -# if (*src == '\\') { -# switch (src[1]) { -# case 'x': return parse_hex(src + 2, 2); -# case 'u': return parse_hex(src + 2, 4); -# case 'U': return parse_hex(src + 2, 8); -# case 't': return std::make_pair('\t', src + 2); -# case 'r': return std::make_pair('\r', src + 2); -# case 'n': return std::make_pair('\n', src + 2); -# case '\\': -# case '"': -# case '[': -# case ']': -# return std::make_pair(src[1], src + 2); -# default: -# throw std::runtime_error(std::string("unknown escape at ") + src); -# } -# } else if (*src) { -# return decode_utf8(src); -# } -# throw std::runtime_error("unexpected end of input"); -# } -def parse_char(src: str) -> typing.Tuple[int, str]: - if not src: - raise ValueError("unexpected end of input") - if src[0] == "\\": - if src[1] == "x": - return parse_hex(src[2:], 2) - elif src[1] == "u": - return parse_hex(src[2:], 4) - elif src[1] == "U": - return parse_hex(src[2:], 8) - elif src[1] == "t": - return ord("\t"), src[2:] - elif src[1] == "r": - return ord("\r"), src[2:] - elif src[1] == "n": - return ord("\n"), src[2:] - elif src[1] in ('\\', '"', '[', ']'): - return ord(src[1]), src[2:] - else: - raise ValueError(f"unknown escape at {src}") - return decode_utf8(src) - -# static const char * parse_sequence( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# std::vector & out_elements, -# bool is_nested) { -# size_t last_sym_start = out_elements.size(); -# const char * pos = src; -# -# auto handle_repetitions = [&](int min_times, int max_times) { -# -# if (last_sym_start == out_elements.size()) { -# throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos); -# } -# -# // apply transformation to previous symbol (last_sym_start to end) according to -# // the following rewrite rules: -# // S{m,n} --> S S S (m times) S'(n-m) -# // S'(x) ::= S S'(x-1) | -# // (... n-m definitions of these S' rules ...) -# // S'(1) ::= S | -# // S{m,} --> S S S (m times) S' -# // S' ::= S S' | -# // S* --> S{0,} -# // --> S' ::= S S' | -# // S+ --> S{1,} -# // --> S S' -# // S' ::= S S' | -# // S? --> S{0,1} -# // --> S' -# // S' ::= S | -# -# std::vector previous_elements(out_elements.begin() + last_sym_start, out_elements.end()); -# if (min_times == 0) { -# out_elements.resize(last_sym_start); -# } else { -# // Repeat the previous elements (min_times - 1) times -# for (int i = 1; i < min_times; i++) { -# out_elements.insert(out_elements.end(), previous_elements.begin(), previous_elements.end()); -# } -# } -# -# uint32_t last_rec_rule_id = 0; -# auto n_opt = max_times < 0 ? 1 : max_times - min_times; -# -# std::vector rec_rule(previous_elements); -# for (int i = 0; i < n_opt; i++) { -# rec_rule.resize(previous_elements.size()); -# uint32_t rec_rule_id = generate_symbol_id(state, rule_name); -# if (i > 0 || max_times < 0) { -# rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, max_times < 0 ? rec_rule_id : last_rec_rule_id}); -# } -# rec_rule.push_back({LLAMA_GRETYPE_ALT, 0}); -# rec_rule.push_back({LLAMA_GRETYPE_END, 0}); -# add_rule(state, rec_rule_id, rec_rule); -# last_rec_rule_id = rec_rule_id; -# } -# if (n_opt > 0) { -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id}); -# } -# }; -# -# while (*pos) { -# if (*pos == '"') { // literal string -# pos++; -# last_sym_start = out_elements.size(); -# while (*pos != '"') { -# if (!*pos) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto char_pair = parse_char(pos); -# pos = char_pair.second; -# out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '[') { // char range(s) -# pos++; -# enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; -# if (*pos == '^') { -# pos++; -# start_type = LLAMA_GRETYPE_CHAR_NOT; -# } -# last_sym_start = out_elements.size(); -# while (*pos != ']') { -# if (!*pos) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto char_pair = parse_char(pos); -# pos = char_pair.second; -# enum llama_gretype type = last_sym_start < out_elements.size() -# ? LLAMA_GRETYPE_CHAR_ALT -# : start_type; -# -# out_elements.push_back({type, char_pair.first}); -# if (pos[0] == '-' && pos[1] != ']') { -# if (!pos[1]) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto endchar_pair = parse_char(pos + 1); -# pos = endchar_pair.second; -# out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); -# } -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (is_word_char(*pos)) { // rule reference -# const char * name_end = parse_name(pos); -# uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); -# pos = parse_space(name_end, is_nested); -# last_sym_start = out_elements.size(); -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); -# } else if (*pos == '(') { // grouping -# // parse nested alternates into synthesized rule -# pos = parse_space(pos + 1, true); -# uint32_t sub_rule_id = generate_symbol_id(state, rule_name); -# pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); -# last_sym_start = out_elements.size(); -# // output reference to synthesized rule -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); -# if (*pos != ')') { -# throw std::runtime_error(std::string("expecting ')' at ") + pos); -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '.') { // any char -# last_sym_start = out_elements.size(); -# out_elements.push_back({LLAMA_GRETYPE_CHAR_ANY, 0}); -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '*') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(0, -1); -# } else if (*pos == '+') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(1, -1); -# } else if (*pos == '?') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(0, 1); -# } else if (*pos == '{') { -# pos = parse_space(pos + 1, is_nested); -# -# if (!is_digit_char(*pos)) { -# throw std::runtime_error(std::string("expecting an int at ") + pos); -# } -# const char * int_end = parse_int(pos); -# int min_times = std::stoul(std::string(pos, int_end - pos)); -# pos = parse_space(int_end, is_nested); -# -# int max_times = -1; -# -# if (*pos == '}') { -# max_times = min_times; -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == ',') { -# pos = parse_space(pos + 1, is_nested); -# -# if (is_digit_char(*pos)) { -# const char * int_end = parse_int(pos); -# max_times = std::stoul(std::string(pos, int_end - pos)); -# pos = parse_space(int_end, is_nested); -# } -# -# if (*pos != '}') { -# throw std::runtime_error(std::string("expecting '}' at ") + pos); -# } -# pos = parse_space(pos + 1, is_nested); -# } else { -# throw std::runtime_error(std::string("expecting ',' at ") + pos); -# } -# handle_repetitions(min_times, max_times); -# } else { -# break; -# } -# } -# return pos; -# } -def parse_sequence(state: ParseState, src: str, rule_name: str, out_elements: typing.List[GrammarElement], is_nested: bool) -> str: - last_sym_start = len(out_elements) - pos = src - - def handle_repetitions(min_times: int, max_times: int) -> None: - nonlocal state, src, rule_name, out_elements, is_nested, last_sym_start, pos - - if last_sym_start == len(out_elements): - raise ValueError(f"expecting preceding item to */+/?/{{ at {pos}") - - previous_elements = out_elements[last_sym_start:] - if min_times == 0: - del out_elements[last_sym_start:] - else: - for i in range(1, min_times): - out_elements.extend(previous_elements) - - last_rec_rule_id = 0 - n_opt = 1 if max_times < 0 else max_times - min_times - - rec_rule = previous_elements[:] - for i in range(n_opt): - rec_rule = rec_rule[:len(previous_elements)] - rec_rule_id = generate_symbol_id(state, rule_name) - if i > 0 or max_times < 0: - rec_rule.append(GrammarElement(GrammarElementType.RULE_REF, rec_rule_id if max_times < 0 else last_rec_rule_id)) - rec_rule.append(GrammarElement(GrammarElementType.ALT, 0)) - rec_rule.append(GrammarElement(GrammarElementType.END, 0)) - add_rule(state, rec_rule_id, rec_rule) - last_rec_rule_id = rec_rule_id - if n_opt > 0: - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, last_rec_rule_id)) - - while pos: - if pos[0] == '"': - pos = pos[1:] - last_sym_start = len(out_elements) - while not pos.startswith('"'): - if not pos: - raise ValueError("unexpected end of input") - char, pos = parse_char(pos) - out_elements.append(GrammarElement(GrammarElementType.CHAR, char)) - pos = parse_space(pos[1:], is_nested) - elif pos[0] == "[": - pos = pos[1:] - start_type = GrammarElementType.CHAR - if pos[0] == "^": - pos = pos[1:] - start_type = GrammarElementType.CHAR_NOT - last_sym_start = len(out_elements) - while pos[0] != "]": - if not pos: - raise ValueError("unexpected end of input") - char, pos = parse_char(pos) - type = GrammarElementType.CHAR_ALT if last_sym_start < len(out_elements) else start_type - out_elements.append(GrammarElement(type, char)) - if pos[0] == "-" and pos[1] != "]": - if not pos[1]: - raise ValueError("unexpected end of input") - endchar, pos = parse_char(pos[1:]) - out_elements.append(GrammarElement(GrammarElementType.CHAR_RNG_UPPER, endchar)) - pos = parse_space(pos[1:], is_nested) - elif pos and is_word_char(pos[0]): - name, rest = parse_name(pos) - ref_rule_id = get_symbol_id(state, name) - pos = parse_space(rest, is_nested) - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, ref_rule_id)) - elif pos.startswith("("): - pos = parse_space(pos[1:], newline_ok=True) - sub_rule_id = generate_symbol_id(state, rule_name) - pos = parse_alternates(state, pos, rule_name, sub_rule_id, is_nested=True) - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, sub_rule_id)) - if pos[0] != ")": - raise ValueError(f"expecting ')' at {pos}") - pos = parse_space(pos[1:], is_nested) - elif pos.startswith("."): - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.CHAR_ANY, 0)) - pos = parse_space(pos[1:], is_nested) - elif pos.startswith("*"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(0, -1) - elif pos.startswith("+"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(1, -1) - elif pos.startswith("?"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(0, 1) - elif pos.startswith("{"): - pos = parse_space(pos[1:], is_nested) - - if not is_digit_char(pos): - raise ValueError(f"expecting an int at {pos}") - min_times, pos = parse_int(pos) - pos = parse_space(pos, is_nested) - - max_times = -1 - - if pos[0] == "}": - max_times = min_times - pos = parse_space(pos[1:], is_nested) - elif pos[0] == ",": - pos = parse_space(pos[1:], is_nested) - - if is_digit_char(pos): - max_times, pos = parse_int(pos) - pos = parse_space(pos, is_nested) - - if pos[0] != "}": - raise ValueError("expecting '}' at {}".format(pos)) - - pos = parse_space(pos[1:], is_nested) - else: - raise ValueError(f"expecting ',' at {pos}") - handle_repetitions(min_times, max_times) - else: - break - return pos - - -# const char * parse_alternates( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# uint32_t rule_id, -# bool is_nested) { -# std::vector rule; -# const char * pos = parse_sequence(state, src, rule_name, rule, is_nested); -# while (*pos == '|') { -# rule.push_back({LLAMA_GRETYPE_ALT, 0}); -# pos = parse_space(pos + 1, true); -# pos = parse_sequence(state, pos, rule_name, rule, is_nested); -# } -# rule.push_back({LLAMA_GRETYPE_END, 0}); -# add_rule(state, rule_id, rule); -# return pos; -# } -def parse_alternates(state: ParseState, src: str, rule_name: str, rule_id: int, is_nested: bool) -> str: - rule = [] - pos = parse_sequence(state, src, rule_name, rule, is_nested) - while pos.startswith("|"): - rule.append(GrammarElement(GrammarElementType.ALT, 0)) - pos = parse_space(pos[1:], newline_ok=True) - pos = parse_sequence(state, pos, rule_name, rule, is_nested) - rule.append(GrammarElement(GrammarElementType.END, 0)) - add_rule(state, rule_id, rule) - return pos - - -# static const char * parse_rule(parse_state & state, const char * src) { -# const char * name_end = parse_name(src); -# const char * pos = parse_space(name_end, false); -# size_t name_len = name_end - src; -# uint32_t rule_id = get_symbol_id(state, src, name_len); -# const std::string name(src, name_len); -# -# if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) { -# throw std::runtime_error(std::string("expecting ::= at ") + pos); -# } -# pos = parse_space(pos + 3, true); -# -# pos = parse_alternates(state, pos, name, rule_id, false); -# -# if (*pos == '\r') { -# pos += pos[1] == '\n' ? 2 : 1; -# } else if (*pos == '\n') { -# pos++; -# } else if (*pos) { -# throw std::runtime_error(std::string("expecting newline or end at ") + pos); -# } -# return parse_space(pos, true); -# } -def parse_rule(state: ParseState, src: str) -> str: - pos = src - name, pos = parse_name(pos) - pos = parse_space(pos, newline_ok=False) - rule_id = get_symbol_id(state, name) - - if not pos.startswith("::="): - raise ValueError(f"expecting ::= at {pos}") - - pos = parse_space(pos[3:], newline_ok=True) - - pos = parse_alternates(state, pos, name, rule_id, is_nested=False) - - if pos.startswith("\r"): - pos = pos[2:] if pos[1] == "\n" else pos[1:] - elif pos.startswith("\n"): - pos = pos[1:] - elif pos: - raise ValueError(f"expecting newline or end at {pos}") - return parse_space(pos, newline_ok=True) - - -# parse_state parse(const char * src) { -# try { -# parse_state state; -# const char * pos = parse_space(src, true); -# while (*pos) { -# pos = parse_rule(state, pos); -# } -# // Validate the state to ensure that all rules are defined -# for (const auto & rule : state.rules) { -# for (const auto & elem : rule) { -# if (elem.type == LLAMA_GRETYPE_RULE_REF) { -# // Ensure that the rule at that location exists -# if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) { -# // Get the name of the rule that is missing -# for (const auto & kv : state.symbol_ids) { -# if (kv.second == elem.value) { -# throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); -# } -# } -# } -# } -# } -# } -# return state; -# } catch (const std::exception & err) { -# fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what()); -# return parse_state(); -# } -# } -def parse(src: str) -> ParseState: - state = ParseState() - pos = src - pos = parse_space(pos, newline_ok=True) - while pos: - pos = parse_rule(state, pos) - # validate - for rule in state.rules: - for elem in rule: - if elem.type == GrammarElementType.RULE_REF: - if elem.value >= len(state.rules) or not state.rules[elem.value]: - for k, v in state.symbol_ids.items(): - if v == elem.value: - raise ValueError(f"Undefined rule identifier '{k}'") - return state - - -# static bool is_char_element(llama_grammar_element elem) { -# switch (elem.type) { -# case LLAMA_GRETYPE_CHAR: return true; -# case LLAMA_GRETYPE_CHAR_NOT: return true; -# case LLAMA_GRETYPE_CHAR_ALT: return true; -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true; -# case LLAMA_GRETYPE_CHAR_ANY: return true; -# default: return false; -# } -# } -def is_char_element(elem: GrammarElement) -> bool: - return elem.type in ( - GrammarElementType.CHAR, - GrammarElementType.CHAR_NOT, - GrammarElementType.CHAR_ALT, - GrammarElementType.CHAR_RNG_UPPER, - GrammarElementType.CHAR_ANY - ) - - -def print_grammar_char(file: typing.TextIO, c: int) -> None: - if 0x20 <= c <= 0x7f: - print(chr(c), end="", file=file) - else: - print(f"", end="", file=file) - - -# static void print_rule( -# FILE * file, -# uint32_t rule_id, -# const std::vector & rule, -# const std::map & symbol_id_names) { -# if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { -# throw std::runtime_error( -# "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); -# } -# fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); -# for (size_t i = 0, end = rule.size() - 1; i < end; i++) { -# llama_grammar_element elem = rule[i]; -# switch (elem.type) { -# case LLAMA_GRETYPE_END: -# throw std::runtime_error( -# "unexpected end of rule: " + std::to_string(rule_id) + "," + -# std::to_string(i)); -# case LLAMA_GRETYPE_ALT: -# fprintf(file, "| "); -# break; -# case LLAMA_GRETYPE_RULE_REF: -# fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); -# break; -# case LLAMA_GRETYPE_CHAR: -# fprintf(file, "["); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_NOT: -# fprintf(file, "[^"); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: -# if (i == 0 || !is_char_element(rule[i - 1])) { -# throw std::runtime_error( -# "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + -# std::to_string(rule_id) + "," + std::to_string(i)); -# } -# fprintf(file, "-"); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_ALT: -# if (i == 0 || !is_char_element(rule[i - 1])) { -# throw std::runtime_error( -# "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + -# std::to_string(rule_id) + "," + std::to_string(i)); -# } -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_ANY: -# fprintf(file, "."); -# break; -# } -# if (is_char_element(elem)) { -# switch (rule[i + 1].type) { -# case LLAMA_GRETYPE_CHAR_ALT: -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: -# case LLAMA_GRETYPE_CHAR_ANY: -# break; -# default: -# fprintf(file, "] "); -# } -# } -# } -# fprintf(file, "\n"); -# } -def print_rule( - file: typing.TextIO, - rule_id: int, - rule: typing.List[GrammarElement], - symbol_id_names: typing.Dict[int, str], -) -> None: - if not rule or rule[-1].type != GrammarElementType.END: - raise ValueError(f"malformed rule, does not end with LLAMA_GRETYPE_END: {rule_id}") - - print(f"{symbol_id_names[rule_id]} ::=", end=" ", file=file) - - for i, elem in enumerate(rule[:-1]): - if elem.type == GrammarElementType.END: - raise ValueError(f"unexpected end of rule: {rule_id}, {i}") - if elem.type == GrammarElementType.ALT: - print("| ", end="", file=file) - elif elem.type == GrammarElementType.RULE_REF: - print(f"{symbol_id_names[elem.value]} ", end="", file=file) - elif elem.type == GrammarElementType.CHAR: - print("[", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_NOT: - print("[^", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_RNG_UPPER: - if i == 0 or not is_char_element(rule[i - 1]): - raise ValueError(f"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: {rule_id}, {i}") - print(f"-", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_ALT: - if i == 0 or not is_char_element(rule[i - 1]): - raise ValueError(f"LLAMA_GRETYPE_CHAR_ALT without preceding char: {rule_id}, {i}") - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_ANY: - print(".", end="", file=file) - if is_char_element(elem): - if rule[i + 1].type in (GrammarElementType.CHAR_ALT, GrammarElementType.CHAR_RNG_UPPER, GrammarElementType.CHAR_ANY): - continue - print("] ", end="", file=file) - print(file=file) - - -def print_grammar(file: typing.TextIO, state: ParseState) -> None: - try: - symbol_id_names = {v: k for k, v in state.symbol_ids.items()} - for i, rule in enumerate(state.rules): - print_rule(file, i, rule, symbol_id_names) - except Exception as err: - print(f"\nerror printing grammar: {err}", file=file) - raise err - +LLAMA_GRAMMAR_DEFAULT_ROOT = "root" class LlamaGrammar: - def __init__(self, parse_state: ParseState): - self.parse_state = parse_state - - self._grammar_rules = parse_state.rules - self._n_rules = len(self._grammar_rules) - self._start_rule_index = parse_state.symbol_ids["root"] - - self._element_lists = [ - [ - llama_cpp.llama_grammar_element(ctypes.c_int(elem.type), ctypes.c_uint32(elem.value)) - for elem in subvector - ] - for subvector in self._grammar_rules - ] - - # Step 2: Convert each list to llama_grammar_element array and get pointer - self._element_arrays = [ - (llama_cpp.llama_grammar_element * len(sublist))(*sublist) - for sublist in self._element_lists - ] - - # Step 3: Get pointer of each array - self._element_array_pointers = [ - ctypes.cast(subarray, llama_cpp.llama_grammar_element_p) for subarray in self._element_arrays - ] - - # Step 4: Make array of these pointers and get its pointer - self._rules = (llama_cpp.llama_grammar_element_p * len(self._element_array_pointers))( - *self._element_array_pointers - ) - - self.grammar = None - self._init_grammar() - - - def _init_grammar(self): - grammar = llama_cpp.llama_grammar_init( - self._rules, ctypes.c_size_t(self._n_rules), ctypes.c_size_t(self._start_rule_index) - ) - - if grammar is None: - raise ValueError("Failed to create grammar") - - self.grammar = grammar - - def __del__(self): - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.grammar = None - - def reset(self): - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self._init_grammar() + def __init__(self, *args, _grammar: str, **kwargs): + self._grammar = _grammar + self._root = LLAMA_GRAMMAR_DEFAULT_ROOT @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": - parsed_grammar = parse(grammar) - if verbose: - print_grammar(file=sys.stdout, state=parsed_grammar) - return cls(parsed_grammar) + return cls(_grammar=grammar) @classmethod def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": diff --git a/pyproject.toml b/pyproject.toml index ce50c673f..9983ef777 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ test = [ "sse-starlette>=1.6.1", "starlette-context>=0.3.6,<0.4", "pydantic-settings>=2.0.1", + "huggingface-hub>=0.23.0" ] dev = [ "black>=23.3.0", diff --git a/tests/test_llama.py b/tests/test_llama.py index 469ef91ca..cf134c2e7 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -1,14 +1,24 @@ import ctypes +import multiprocessing import numpy as np -import pytest from scipy.special import log_softmax +from huggingface_hub import hf_hub_download + +import pytest + import llama_cpp +import llama_cpp._internals as internals + MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf" +def test_llama_cpp_version(): + assert llama_cpp.__version__ + + def test_llama_cpp_tokenization(): llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, verbose=False) @@ -47,247 +57,117 @@ def test_llama_cpp_tokenization(): @pytest.fixture -def mock_llama(monkeypatch): - def setup_mock(llama: llama_cpp.Llama, output_text: str): - n_ctx = llama.n_ctx() - n_vocab = llama.n_vocab() - output_tokens = llama.tokenize( - output_text.encode("utf-8"), add_bos=True, special=True - ) - logits = (ctypes.c_float * (n_vocab * n_ctx))(-100.0) - for i in range(n_ctx): - output_idx = i + 1 # logits for first tokens predict second token - if output_idx < len(output_tokens): - logits[i * n_vocab + output_tokens[output_idx]] = 100.0 - else: - logits[i * n_vocab + llama.token_eos()] = 100.0 - n = 0 - last_n_tokens = 0 - - def mock_decode(ctx: llama_cpp.llama_context_p, batch: llama_cpp.llama_batch): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - assert batch.n_tokens > 0, "no tokens in batch" - assert all( - batch.n_seq_id[i] == 1 for i in range(batch.n_tokens) - ), "n_seq >1 not supported by mock_llama" - assert all( - batch.seq_id[i][0] == 0 for i in range(batch.n_tokens) - ), "n_seq >1 not supported by mock_llama" - assert batch.logits[ - batch.n_tokens - 1 - ], "logits not allocated for last token" - # Update the mock context state - nonlocal n - nonlocal last_n_tokens - n = max(batch.pos[i] for i in range(batch.n_tokens)) + 1 - last_n_tokens = batch.n_tokens - return 0 - - def mock_get_logits(ctx: llama_cpp.llama_context_p): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - assert n > 0, "mock_llama_decode not called" - assert last_n_tokens > 0, "mock_llama_decode not called" - # Return view of logits for last_n_tokens - return (ctypes.c_float * (last_n_tokens * n_vocab)).from_address( - ctypes.addressof(logits) - + (n - last_n_tokens) * n_vocab * ctypes.sizeof(ctypes.c_float) - ) - - monkeypatch.setattr("llama_cpp.llama_cpp.llama_decode", mock_decode) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_get_logits", mock_get_logits) - - def mock_kv_cache_clear(ctx: llama_cpp.llama_context_p): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_rm( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_cp( - ctx: llama_cpp.llama_context_p, - seq_id_src: llama_cpp.llama_seq_id, - seq_id_dst: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_keep( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_add( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_clear", mock_kv_cache_clear) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_rm", mock_kv_cache_seq_rm) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_cp", mock_kv_cache_seq_cp) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_keep", mock_kv_cache_seq_keep) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_add", mock_kv_cache_seq_add) - - return setup_mock - - -def test_llama_patch(mock_llama): - n_ctx = 128 - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, n_ctx=n_ctx) - n_vocab = llama_cpp.llama_n_vocab(llama._model.model) - assert n_vocab == 32000 - - text = "The quick brown fox" - output_text = " jumps over the lazy dog." - all_text = text + output_text - - ## Test basic completion from bos until eos - mock_llama(llama, all_text) - completion = llama.create_completion("", max_tokens=36) - assert completion["choices"][0]["text"] == all_text - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until eos - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=20) - assert completion["choices"][0]["text"] == output_text - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test streaming completion until eos - mock_llama(llama, all_text) - chunks = list(llama.create_completion(text, max_tokens=20, stream=True)) - assert "".join(chunk["choices"][0]["text"] for chunk in chunks) == output_text - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until stop sequence - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=20, stop=["lazy"]) - assert completion["choices"][0]["text"] == " jumps over the " - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test streaming completion until stop sequence - mock_llama(llama, all_text) - chunks = list( - llama.create_completion(text, max_tokens=20, stream=True, stop=["lazy"]) - ) - assert ( - "".join(chunk["choices"][0]["text"] for chunk in chunks) == " jumps over the " +def llama_cpp_model_path(): + repo_id = "Qwen/Qwen2-0.5B-Instruct-GGUF" + filename = "qwen2-0_5b-instruct-q8_0.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + +def test_real_model(llama_cpp_model_path): + import os + assert os.path.exists(llama_cpp_model_path) + + params = llama_cpp.llama_model_default_params() + params.use_mmap = llama_cpp.llama_supports_mmap() + params.use_mlock = llama_cpp.llama_supports_mlock() + params.check_tensors = False + + model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) + + cparams = llama_cpp.llama_context_default_params() + cparams.n_ctx = 16 + cparams.n_batch = 16 + cparams.n_ubatch = 16 + cparams.n_threads = multiprocessing.cpu_count() + cparams.n_threads_batch = multiprocessing.cpu_count() + cparams.logits_all = False + cparams.flash_attn = True + + context = internals.LlamaContext(model=model, params=cparams) + tokens = model.tokenize(b"Hello, world!", add_bos=True, special=True) + + assert tokens == [9707, 11, 1879, 0] + + tokens = model.tokenize(b"The quick brown fox jumps", add_bos=True, special=True) + + batch = internals.LlamaBatch(n_tokens=len(tokens), embd=0, n_seq_max=1) + + seed = 1337 + sampler = internals.LlamaSampler() + sampler.add_top_k(50) + sampler.add_top_p(0.9, 1) + sampler.add_temp(0.8) + sampler.add_dist(seed) + + result = tokens + n_eval = 0 + for _ in range(4): + batch.set_batch(tokens, n_past=n_eval, logits_all=False) + context.decode(batch) + n_eval += len(tokens) + token_id = sampler.sample(context, -1) + tokens = [token_id] + result += tokens + + output = result[5:] + output_text = model.detokenize(output, special=True) + assert output_text == b" over the lazy dog" + +def test_real_llama(llama_cpp_model_path): + model = llama_cpp.Llama( + llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + flash_attn=True, ) - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until length - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=2) - assert completion["choices"][0]["text"] == " jumps" - assert completion["choices"][0]["finish_reason"] == "length" - - ## Test streaming completion until length - mock_llama(llama, all_text) - chunks = list(llama.create_completion(text, max_tokens=2, stream=True)) - assert "".join(chunk["choices"][0]["text"] for chunk in chunks) == " jumps" - assert chunks[-1]["choices"][0]["finish_reason"] == "length" - - -def test_llama_pickle(): - import pickle - import tempfile - - fp = tempfile.TemporaryFile() - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True) - pickle.dump(llama, fp) - fp.seek(0) - llama = pickle.load(fp) - - assert llama - assert llama.ctx is not None - - text = b"Hello World" - - assert llama.detokenize(llama.tokenize(text)) == text - - -def test_utf8(mock_llama): - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, logits_all=True) - - output_text = "😀" - - ## Test basic completion with utf8 multibyte - mock_llama(llama, output_text) - completion = llama.create_completion("", max_tokens=4) - assert completion["choices"][0]["text"] == output_text - - ## Test basic completion with incomplete utf8 multibyte - mock_llama(llama, output_text) - completion = llama.create_completion("", max_tokens=1) - assert completion["choices"][0]["text"] == "" + output = model.create_completion( + "The quick brown fox jumps", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337 + ) + assert output["choices"][0]["text"] == " over the lazy dog" + + + output = model.create_completion( + "The capital of france is paris, 'true' or 'false'?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "true" | "false" +""") + ) + assert output["choices"][0]["text"] == "true" -def test_llama_server(): - from fastapi.testclient import TestClient - from llama_cpp.server.app import create_app, Settings + suffix = b"rot" + tokens = model.tokenize(suffix, add_bos=True, special=True) + def logit_processor_func(input_ids, logits): + for token in tokens: + logits[token] *= 1000 + return logits - settings = Settings( - model=MODEL, - vocab_only=True, + logit_processors = llama_cpp.LogitsProcessorList( + [logit_processor_func] ) - app = create_app(settings) - client = TestClient(app) - response = client.get("/v1/models") - assert response.json() == { - "object": "list", - "data": [ - { - "id": MODEL, - "object": "model", - "owned_by": "me", - "permissions": [], - } - ], - } - - -@pytest.mark.parametrize( - "size_and_axis", - [ - ((32_000,), -1), # last token's next-token logits - ((10, 32_000), -1), # many tokens' next-token logits, or batch of last tokens - ((4, 10, 32_000), -1), # batch of texts - ], -) -@pytest.mark.parametrize("convert_to_list", [True, False]) -def test_logits_to_logprobs(size_and_axis, convert_to_list: bool, atol: float = 1e-7): - size, axis = size_and_axis - logits: np.ndarray = -np.random.uniform(low=0, high=60, size=size) - logits = logits.astype(np.single) - if convert_to_list: - # Currently, logits are converted from arrays to lists. This may change soon - logits = logits.tolist() - log_probs = llama_cpp.Llama.logits_to_logprobs(logits, axis=axis) - log_probs_correct = log_softmax(logits, axis=axis) - assert log_probs.dtype == np.single - assert log_probs.shape == size - assert np.allclose(log_probs, log_probs_correct, atol=atol) - -def test_llama_cpp_version(): - assert llama_cpp.__version__ + output = model.create_completion( + "The capital of france is par", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337, + logits_processor=logit_processors + ) + assert output["choices"][0]["text"].lower().startswith("rot") diff --git a/tests/test_llama_grammar.py b/tests/test_llama_grammar.py index cb221880a..34ef2874d 100644 --- a/tests/test_llama_grammar.py +++ b/tests/test_llama_grammar.py @@ -10,9 +10,9 @@ def test_grammar_from_string(): grammar = llama_cpp.LlamaGrammar.from_string(tree) - assert grammar._n_rules == 3 - assert grammar._start_rule_index == 2 - assert grammar.grammar is not None + # assert grammar._n_rules == 3 + # assert grammar._start_rule_index == 2 + # assert grammar.grammar is not None def test_composed_pydantic_grammar(): @@ -49,7 +49,7 @@ class B(BaseModel): grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(schema)) - assert grammar.grammar is not None + # assert grammar.grammar is not None def test_grammar_anyof(): @@ -75,4 +75,4 @@ def test_grammar_anyof(): grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(sch)) - assert grammar.grammar is not None \ No newline at end of file + # assert grammar.grammar is not None diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8ebe8ddeb..6262d13e0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8ebe8ddebd68526757c631cd019de009697c63c2 +Subproject commit 6262d13e0b2da91f230129a93a996609a2f5a2f2 From 1e64664e0facb7e3595efdf4716f01527f4047ba Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 20:07:48 -0400 Subject: [PATCH 073/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 227 +++++++++++++++++++++++++++-------------- vendor/llama.cpp | 2 +- 2 files changed, 153 insertions(+), 76 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 261d7c6b4..6b82753e1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1161,7 +1161,8 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): ... +def llama_numa_init(numa: int, /): + ... # // Optional: an auto threadpool gets created in ggml if not passed explicitly @@ -1196,7 +1197,8 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... +) -> Optional[llama_model_p]: + ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1205,7 +1207,8 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): ... +def llama_free_model(model: llama_model_p, /): + ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1218,7 +1221,8 @@ def llama_free_model(model: llama_model_p, /): ... ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... +) -> Optional[llama_context_p]: + ... # // Frees all allocated memory @@ -1239,87 +1243,110 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: ... +def llama_time_us() -> int: + ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: ... +def llama_max_devices() -> int: + ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: ... +def llama_supports_mmap() -> bool: + ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: ... +def llama_supports_mlock() -> bool: + ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: ... +def llama_supports_gpu_offload() -> bool: + ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: ... +def llama_n_batch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: + ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: ... +def llama_n_vocab(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... +def llama_n_embd(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... +def llama_n_layer(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_n_head (const struct llama_model * model); +@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) +def llama_n_head(model: llama_model_p, /) -> int: + ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: + ... # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: ... +def llama_vocab_type(model: llama_model_p, /) -> int: + ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: ... +def llama_rope_type(model: llama_model_p, /) -> int: + ... # // Get the model's RoPE frequency scaling factor @@ -2110,7 +2137,8 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2138,7 +2166,8 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # LLAMA_API bool llama_state_save_file( @@ -2162,7 +2191,8 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2187,7 +2217,8 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2285,7 +2316,8 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2315,7 +2347,8 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # // @@ -2620,7 +2653,8 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: ... +) -> bytes: + ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2629,7 +2663,8 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: ... +) -> float: + ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2638,7 +2673,8 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: ... +) -> int: + ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2703,12 +2739,14 @@ def llama_token_nl(model: llama_model_p, /) -> int: # LLAMA_API bool llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: ... +def llama_add_bos_token(model: llama_model_p, /) -> bool: + ... # LLAMA_API bool llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: ... +def llama_add_eos_token(model: llama_model_p, /) -> bool: + ... # // Codellama infill tokens @@ -2721,17 +2759,20 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: ... +def llama_token_middle(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: ... +def llama_token_suffix(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: ... +def llama_token_eot(model: llama_model_p, /) -> int: + ... # // @@ -2932,7 +2973,8 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: ... +) -> int: + ... # // @@ -2991,7 +3033,8 @@ def llama_chat_apply_template( # // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph # //void (*apply_ggml) (struct llama_sampler * smpl, ...); # }; -class llama_sampler_i(ctypes.Structure): ... +class llama_sampler_i(ctypes.Structure): + ... # struct llama_sampler { @@ -3036,7 +3079,8 @@ class llama_sampler(ctypes.Structure): [llama_sampler_p_ctypes], ctypes.c_char_p, ) -def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... +def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: + ... # LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); @@ -3045,7 +3089,8 @@ def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... [llama_sampler_p_ctypes, llama_token], None, ) -def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ... +def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): + ... # LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); @@ -3056,7 +3101,8 @@ def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], ) def llama_sampler_apply( smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], / -): ... +): + ... # LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); @@ -3065,7 +3111,8 @@ def llama_sampler_apply( [llama_sampler_p_ctypes], None, ) -def llama_sampler_reset(smpl: llama_sampler_p, /): ... +def llama_sampler_reset(smpl: llama_sampler_p, /): + ... # LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); @@ -3074,7 +3121,8 @@ def llama_sampler_reset(smpl: llama_sampler_p, /): ... [llama_sampler_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: + ... # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) @@ -3084,7 +3132,8 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... [llama_sampler_p_ctypes], None, ) -def llama_sampler_free(smpl: llama_sampler_p, /): ... +def llama_sampler_free(smpl: llama_sampler_p, /): + ... # // llama_sampler_chain @@ -3096,9 +3145,8 @@ def llama_sampler_free(smpl: llama_sampler_p, /): ... [llama_sampler_chain_params], llama_sampler_p_ctypes, ) -def llama_sampler_chain_init( - params: llama_sampler_chain_params, / -) -> llama_sampler_p: ... +def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sampler_p: + ... # // important: takes ownership of the sampler object and will free it when llama_sampler_free is called @@ -3108,7 +3156,8 @@ def llama_sampler_chain_init( [llama_sampler_p_ctypes, llama_sampler_p_ctypes], None, ) -def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ... +def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): + ... # LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i); @@ -3119,7 +3168,8 @@ def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): . ) def llama_sampler_chain_get( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); @@ -3128,7 +3178,8 @@ def llama_sampler_chain_get( [llama_sampler_p_ctypes], ctypes.c_int, ) -def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... +def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: + ... # // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed @@ -3140,31 +3191,36 @@ def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... ) def llama_sampler_chain_remove( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # // available samplers: # # LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) -def llama_sampler_init_greedy() -> llama_sampler_p: ... +def llama_sampler_init_greedy() -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); @ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) -def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... +def llama_sampler_init_dist(seed: int) -> llama_sampler_p: + ... # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. # LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) -def llama_sampler_init_softmax() -> llama_sampler_p: ... +def llama_sampler_init_softmax() -> llama_sampler_p: + ... # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) -def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... +def llama_sampler_init_top_k(k: int) -> llama_sampler_p: + ... # /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 @@ -3174,7 +3230,8 @@ def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 @@ -3184,7 +3241,8 @@ def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. @@ -3194,7 +3252,8 @@ def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. @@ -3204,12 +3263,14 @@ def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: .. [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); @ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) -def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... +def llama_sampler_init_temp(t: float) -> llama_sampler_p: + ... # /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. @@ -3221,7 +3282,8 @@ def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... ) def llama_sampler_init_temp_ext( t: float, delta: float, exponent: float -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3243,7 +3305,8 @@ def llama_sampler_init_temp_ext( ) def llama_sampler_init_mirostat( n_vocab: int, seed: int, tau: float, eta: float, m: int, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3262,7 +3325,8 @@ def llama_sampler_init_mirostat( ) def llama_sampler_init_mirostat_v2( seed: int, tau: float, eta: float, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( @@ -3276,7 +3340,8 @@ def llama_sampler_init_mirostat_v2( ) def llama_sampler_init_grammar( model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( @@ -3315,7 +3380,8 @@ def llama_sampler_init_penalties( penalize_nl: bool, ignore_eos: bool, /, -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( @@ -3329,7 +3395,8 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_logit_bias( n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise @@ -3339,7 +3406,8 @@ def llama_sampler_init_logit_bias( [llama_sampler_p_ctypes], ctypes.c_uint32, ) -def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... +def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: + ... # /// @details Sample and accept a token from the idx-th output of the last evaluation @@ -3360,7 +3428,8 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... ) def llama_sampler_sample( smpl: llama_sampler_p, ctx: llama_context_p, idx: int, / -) -> int: ... +) -> int: + ... # // @@ -3413,7 +3482,8 @@ def llama_split_prefix( # // Print system information # LLAMA_API const char * llama_print_system_info(void); @ctypes_function("llama_print_system_info", [], ctypes.c_char_p) -def llama_print_system_info() -> bytes: ... +def llama_print_system_info() -> bytes: + ... # // Set callback for all future logging events. @@ -3480,7 +3550,8 @@ class llama_perf_sampler_data(ctypes.Structure): [llama_context_p_ctypes], llama_perf_context_data, ) -def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... +def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: + ... # LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); @@ -3489,7 +3560,8 @@ def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... [llama_context_p_ctypes], None, ) -def llama_perf_context_print(ctx: llama_context_p, /): ... +def llama_perf_context_print(ctx: llama_context_p, /): + ... # LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); @@ -3498,7 +3570,8 @@ def llama_perf_context_print(ctx: llama_context_p, /): ... [llama_context_p_ctypes], None, ) -def llama_perf_context_reset(ctx: llama_context_p, /): ... +def llama_perf_context_reset(ctx: llama_context_p, /): + ... # // NOTE: the following work only with samplers constructed via llama_sampler_chain_init @@ -3508,7 +3581,8 @@ def llama_perf_context_reset(ctx: llama_context_p, /): ... [llama_sampler_p_ctypes], llama_perf_sampler_data, ) -def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ... +def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: + ... # LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); @@ -3517,7 +3591,8 @@ def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: .. [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_print(chain: llama_sampler_p, /): ... +def llama_perf_sampler_print(chain: llama_sampler_p, /): + ... # LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); @@ -3526,7 +3601,8 @@ def llama_perf_sampler_print(chain: llama_sampler_p, /): ... [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... +def llama_perf_sampler_reset(chain: llama_sampler_p, /): + ... # LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); @@ -3537,4 +3613,5 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... ) def llama_perf_dump_yaml( stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / -): ... +): + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6262d13e0..64c6af319 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6262d13e0b2da91f230129a93a996609a2f5a2f2 +Subproject commit 64c6af3195c3cd4aa3328a1282d29cd2635c34c9 From 9b64bb5b137385cdf535598df5b2c34ed459450f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 20:11:02 -0400 Subject: [PATCH 074/155] misc: Format --- llama_cpp/_internals.py | 66 ++++++++++++++++++------ llama_cpp/llama.py | 98 ++++++++++++++++++++---------------- llama_cpp/llama_cache.py | 6 +-- llama_cpp/llama_grammar.py | 3 +- llama_cpp/llama_tokenizer.py | 33 ++++++++---- llama_cpp/llava_cpp.py | 21 +++++--- 6 files changed, 147 insertions(+), 80 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 6475695c6..0aff34844 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -100,7 +100,6 @@ def n_params(self) -> int: def get_tensor(self, name: str) -> ctypes.c_void_p: return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) - # Vocab def token_get_text(self, token: int) -> str: @@ -460,9 +459,7 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - batch = llama_cpp.llama_batch_init( - self._n_tokens, self.embd, self.n_seq_max - ) + batch = llama_cpp.llama_batch_init(self._n_tokens, self.embd, self.n_seq_max) if batch is None: raise ValueError("Failed to create llama_batch") @@ -541,6 +538,7 @@ def copy_logits(self, logits: npt.NDArray[np.single]): # Embedding functions + def normalize_embedding(embedding): norm = float(np.linalg.norm(embedding)) if norm == 0.0: @@ -713,11 +711,17 @@ def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): import ctypes import llama_cpp + class CustomSampler: - def __init__(self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None]): + def __init__( + self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None] + ): self.apply_func = apply_func - def apply_wrapper(sampler: llama_cpp.llama_sampler_p, cur_p: llama_cpp.llama_token_data_array_p): + def apply_wrapper( + sampler: llama_cpp.llama_sampler_p, + cur_p: llama_cpp.llama_token_data_array_p, + ): self.apply_func(cur_p) def free_wrapper(sampler: llama_cpp.llama_sampler_p): @@ -740,6 +744,7 @@ def free_wrapper(sampler: llama_cpp.llama_sampler_p): def get_sampler(self) -> llama_cpp.llama_sampler_p: return ctypes.pointer(self.sampler) + class LlamaSampler: def __init__(self): params = llama_cpp.llama_sampler_chain_params() @@ -788,9 +793,7 @@ def add_temp_ext(self, t: float, delta: float, exponent: float): self._add_sampler(sampler) def add_mirostat(self, n_vocab: int, seed: int, tau: float, eta: float, m: int): - sampler = llama_cpp.llama_sampler_init_mirostat( - n_vocab, seed, tau, eta, m - ) + sampler = llama_cpp.llama_sampler_init_mirostat(n_vocab, seed, tau, eta, m) self._add_sampler(sampler) def add_mirostat_v2(self, seed: int, tau: float, eta: float): @@ -798,23 +801,54 @@ def add_mirostat_v2(self, seed: int, tau: float, eta: float): self._add_sampler(sampler) def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): - sampler = llama_cpp.llama_sampler_init_grammar(model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8")) + sampler = llama_cpp.llama_sampler_init_grammar( + model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") + ) self._add_sampler(sampler) - def add_penalties(self, n_vocab: int, special_eos_id: int, linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, penalize_nl: bool, ignore_eos: bool): - sampler = llama_cpp.llama_sampler_init_penalties(n_vocab, special_eos_id, linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, penalize_nl, ignore_eos) + def add_penalties( + self, + n_vocab: int, + special_eos_id: int, + linefeed_id: int, + penalty_last_n: int, + penalty_repeat: float, + penalty_freq: float, + penalty_present: float, + penalize_nl: bool, + ignore_eos: bool, + ): + sampler = llama_cpp.llama_sampler_init_penalties( + n_vocab, + special_eos_id, + linefeed_id, + penalty_last_n, + penalty_repeat, + penalty_freq, + penalty_present, + penalize_nl, + ignore_eos, + ) self._add_sampler(sampler) - def init_logit_bias(self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p): - sampler = llama_cpp.llama_sampler_init_logit_bias(n_vocab, n_logit_bias, logit_bias) + def init_logit_bias( + self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p + ): + sampler = llama_cpp.llama_sampler_init_logit_bias( + n_vocab, n_logit_bias, logit_bias + ) self._add_sampler(sampler) - def add_custom(self, apply_func: Callable[[llama_cpp.llama_token_data_array], None]): + def add_custom( + self, apply_func: Callable[[llama_cpp.llama_token_data_array], None] + ): custom_sampler = CustomSampler(apply_func) sampler = custom_sampler.get_sampler() self._add_sampler(sampler) # NOTE: Must remove custom samplers before free or llama.cpp will try to free them - self.custom_samplers.append((llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler)) + self.custom_samplers.append( + (llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler) + ) def _add_sampler(self, sampler: llama_cpp.llama_sampler_p): assert self.sampler is not None diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 557134a6e..7dd862515 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -255,28 +255,28 @@ def __init__( for i, (k, v) in enumerate(kv_overrides.items()): self._kv_overrides_array[i].key = k.encode("utf-8") if isinstance(v, bool): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL self._kv_overrides_array[i].value.val_bool = v elif isinstance(v, int): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT self._kv_overrides_array[i].value.val_i64 = v elif isinstance(v, float): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT self._kv_overrides_array[i].value.val_f64 = v elif isinstance(v, str): # type: ignore v_bytes = v.encode("utf-8") if len(v_bytes) > 128: # TODO: Make this a constant raise ValueError(f"Value for {k} is too long: {v}") v_bytes = v_bytes.ljust(128, b"\0") - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR # copy min(v_bytes, 128) to str_value address = typing.cast( int, @@ -292,9 +292,9 @@ def __init__( else: raise ValueError(f"Unknown value type for {k}: {v}") - self._kv_overrides_array[-1].key = ( - b"\0" # ensure sentinel element is zeroed - ) + self._kv_overrides_array[ + -1 + ].key = b"\0" # ensure sentinel element is zeroed self.model_params.kv_overrides = self._kv_overrides_array self.n_batch = min(n_ctx, n_batch) # ??? @@ -431,9 +431,9 @@ def free_lora_adapter(): self.chat_format = chat_format self.chat_handler = chat_handler - self._chat_handlers: Dict[str, llama_chat_format.LlamaChatCompletionHandler] = ( - {} - ) + self._chat_handlers: Dict[ + str, llama_chat_format.LlamaChatCompletionHandler + ] = {} self.draft_model = draft_model @@ -580,7 +580,10 @@ def tokenize( return self.tokenizer_.tokenize(text, add_bos, special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: """Detokenize a list of tokens. @@ -592,7 +595,9 @@ def detokenize( Returns: The detokenized string. """ - return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens, special=special) + return self.tokenizer_.detokenize( + tokens, prev_tokens=prev_tokens, special=special + ) def set_cache(self, cache: Optional[BaseLlamaCache]): """Set the cache. @@ -681,12 +686,16 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): recarray = np.recarray( shape=(size,), dtype=np.dtype( - [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True + [("id", np.intc), ("logit", np.single), ("p", np.single)], + align=True, + ), + buf=(llama_cpp.llama_token_data * size).from_address( + data_soa_address ), - buf=(llama_cpp.llama_token_data * size).from_address(data_soa_address), ) for logit_processor in logits_processor: recarray.logit[:] = logit_processor(self._input_ids, recarray.logit) + sampler.add_custom(apply_func) sampler.add_penalties( @@ -698,7 +707,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): penalty_freq=frequency_penalty, penalty_present=presence_penalty, penalize_nl=penalize_nl, - ignore_eos=False + ignore_eos=False, ) if grammar is not None: @@ -841,22 +850,22 @@ def generate( # Reset mirostat sampling self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau) self._sampler = self._init_sampler( - top_k=top_k, - top_p=top_p, - min_p=min_p, - typical_p=typical_p, - temp=temp, - repeat_penalty=repeat_penalty, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - tfs_z=tfs_z, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - penalize_nl=penalize_nl, - logits_processor=logits_processor, - grammar=grammar, - seed=seed, + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, + seed=seed, ) # Check for kv cache prefix match @@ -872,8 +881,11 @@ def generate( tokens = tokens[longest_prefix:] self.n_tokens = longest_prefix if self.verbose: - print(f"Llama.generate: {longest_prefix} prefix-match hit, " - f"remaining {len(tokens)} prompt tokens to eval", file=sys.stderr) + print( + f"Llama.generate: {longest_prefix} prefix-match hit, " + f"remaining {len(tokens)} prompt tokens to eval", + file=sys.stderr, + ) # Reset the model state if reset: @@ -1032,7 +1044,9 @@ def decode_batch(seq_sizes: List[int]): for j in range(size) ] if normalize: - embedding = [internals.normalize_embedding(e) for e in embedding] + embedding = [ + internals.normalize_embedding(e) for e in embedding + ] data.append(embedding) pos += size else: diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index 5220c7933..e059e98e1 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -52,9 +52,9 @@ class LlamaRAMCache(BaseLlamaCache): def __init__(self, capacity_bytes: int = (2 << 30)): super().__init__(capacity_bytes) self.capacity_bytes = capacity_bytes - self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = ( - OrderedDict() - ) + self.cache_state: OrderedDict[ + Tuple[int, ...], "llama_cpp.llama.LlamaState" + ] = OrderedDict() @property def cache_size(self): diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 785aa88d7..b95c77ab5 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -15,6 +15,7 @@ LLAMA_GRAMMAR_DEFAULT_ROOT = "root" + class LlamaGrammar: def __init__(self, *args, _grammar: str, **kwargs): self._grammar = _grammar @@ -23,7 +24,7 @@ def __init__(self, *args, _grammar: str, **kwargs): @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": return cls(_grammar=grammar) - + @classmethod def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": try: diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 2e7590d14..1375e1392 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -27,7 +27,10 @@ def tokenize( @abc.abstractmethod def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: """Detokenize the tokens into text. @@ -49,7 +52,10 @@ def tokenize( return self._model.tokenize(text, add_bos=add_bos, special=special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: return self._model.detokenize(tokens, special=special) @@ -80,19 +86,24 @@ def tokenize( ) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: - skip_special_tokens = not special + skip_special_tokens = not special if prev_tokens is not None: - text = self.hf_tokenizer.decode(prev_tokens + tokens, skip_special_tokens=skip_special_tokens).encode( - "utf-8", errors="ignore" - ) - prev_text = self.hf_tokenizer.decode(prev_tokens, skip_special_tokens=skip_special_tokens).encode( - "utf-8", errors="ignore" - ) + text = self.hf_tokenizer.decode( + prev_tokens + tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") + prev_text = self.hf_tokenizer.decode( + prev_tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") return text[len(prev_text) :] else: - return self.hf_tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens).encode("utf-8", errors="ignore") + return self.hf_tokenizer.decode( + tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") @classmethod def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer": diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index b80d85913..7d97dc0fd 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -165,7 +165,8 @@ class llava_image_embed(Structure): ) def llava_validate_embed_size( ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, / -) -> bool: ... +) -> bool: + ... # /** build an image embed from image file bytes */ @@ -181,7 +182,8 @@ def llava_image_embed_make_with_bytes( image_bytes: CtypesArray[c_uint8], image_bytes_length: Union[c_int, int], /, -) -> "_Pointer[llava_image_embed]": ... +) -> "_Pointer[llava_image_embed]": + ... # /** build an image embed from a path to an image filename */ @@ -193,13 +195,15 @@ def llava_image_embed_make_with_bytes( ) def llava_image_embed_make_with_filename( ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, / -) -> "_Pointer[llava_image_embed]": ... +) -> "_Pointer[llava_image_embed]": + ... # LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed); # /** free an embedding made with llava_image_embed_make_* */ @ctypes_function("llava_image_embed_free", [POINTER(llava_image_embed)], None) -def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): ... +def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): + ... # /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */ @@ -220,7 +224,8 @@ def llava_eval_image_embed( n_batch: Union[c_int, int], n_past: "_Pointer[c_int]", /, -) -> bool: ... +) -> bool: + ... ################################################ @@ -233,10 +238,12 @@ def llava_eval_image_embed( @ctypes_function("clip_model_load", [c_char_p, c_int], clip_ctx_p_ctypes) def clip_model_load( fname: bytes, verbosity: Union[c_int, int], / -) -> Optional[clip_ctx_p]: ... +) -> Optional[clip_ctx_p]: + ... # /** free mmproj model */ # CLIP_API void clip_free(struct clip_ctx * ctx); @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) -def clip_free(ctx: clip_ctx_p, /): ... +def clip_free(ctx: clip_ctx_p, /): + ... From 22cedad8a9f010bdea6186ee564da7aaa21b6684 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Thu, 19 Sep 2024 10:11:01 +0800 Subject: [PATCH 075/155] fix: Fix memory allocation of ndarray (#1704) * Fix memory allocation of ndarray * Add basic LlamaState tests * Improve LlamaState test and fix rng / seed --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 34 +++++++++++++++++++--------------- tests/test_llama.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7dd862515..80fadffe2 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -7,6 +7,7 @@ import json import ctypes import typing +import random import fnmatch import warnings import contextlib @@ -301,9 +302,11 @@ def __init__( self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() + # Used by the sampler + self._seed = seed or llama_cpp.LLAMA_DEFAULT_SEED + # Context Params self.context_params = llama_cpp.llama_context_default_params() - self.context_params.seed = seed self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch self.context_params.n_threads = self.n_threads @@ -613,8 +616,7 @@ def set_seed(self, seed: int): Args: seed: The random seed. """ - # TODO: Fix this - # llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) + self._seed = seed def reset(self): """Reset the model state.""" @@ -672,7 +674,6 @@ def _init_sampler( penalize_nl: bool = True, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - seed: Optional[int] = None, ): sampler = internals.LlamaSampler() @@ -715,7 +716,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): if temp < 0.0: sampler.add_softmax() - sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + sampler.add_dist(self._seed) elif temp == 0.0: sampler.add_greedy() else: @@ -723,14 +724,14 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): mirostat_m = 100 sampler.add_mirostat( self._n_vocab, - seed or llama_cpp.LLAMA_DEFAULT_SEED, + self._seed, mirostat_tau, mirostat_eta, mirostat_m, ) elif mirostat_mode == 2: sampler.add_mirostat_v2( - seed or llama_cpp.LLAMA_DEFAULT_SEED, + self._seed, mirostat_tau, mirostat_eta, ) @@ -743,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_top_p(top_p, min_keep) sampler.add_min_p(min_p, min_keep) sampler.add_temp(temp) - sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + sampler.add_dist(self._seed) return sampler def sample( @@ -826,7 +827,6 @@ def generate( logits_processor: Optional[LogitsProcessorList] = None, stopping_criteria: Optional[StoppingCriteriaList] = None, grammar: Optional[LlamaGrammar] = None, - seed: Optional[int] = None, ) -> Generator[int, Optional[Sequence[int]], None]: """Create a generator of tokens from a prompt. @@ -865,7 +865,6 @@ def generate( penalize_nl=penalize_nl, logits_processor=logits_processor, grammar=grammar, - seed=seed, ) # Check for kv cache prefix match @@ -1301,9 +1300,10 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache miss", file=sys.stderr) - # TODO: Fix this - # if seed is not None: - # self._ctx.set_rng_seed(seed) + if seed is not None: + self.set_seed(seed) + else: + self.set_seed(random.Random(self._seed).randint(0, 2 ** 32)) finish_reason = "length" multibyte_fix = 0 @@ -1324,7 +1324,6 @@ def logit_bias_processor( stopping_criteria=stopping_criteria, logits_processor=logits_processor, grammar=grammar, - seed=seed, ): if llama_cpp.llama_token_is_eog(self._model.model, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) @@ -2136,14 +2135,17 @@ def save_state(self) -> LlamaState: n_tokens=self.n_tokens, llama_state=bytes(llama_state_compact), llama_state_size=n_bytes, + seed=self._seed, ) def load_state(self, state: LlamaState) -> None: # Only filling in up to `n_tokens` and then zero-ing out the rest self.scores[: state.n_tokens, :] = state.scores.copy() - self.scores[state.n_tokens :, :] = 0.0 + rest = self.scores[state.n_tokens :, :] + rest[rest > 0] = 0.0 self.input_ids = state.input_ids.copy() self.n_tokens = state.n_tokens + self._seed = state.seed state_size = state.llama_state_size LLamaStateArrayType = ctypes.c_uint8 * state_size llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state) @@ -2321,12 +2323,14 @@ def __init__( n_tokens: int, llama_state: bytes, llama_state_size: int, + seed: int, ): self.input_ids = input_ids self.scores = scores self.n_tokens = n_tokens self.llama_state = llama_state self.llama_state_size = llama_state_size + self.seed = seed LogitsProcessor = Callable[ diff --git a/tests/test_llama.py b/tests/test_llama.py index cf134c2e7..fc182ae20 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -171,3 +171,48 @@ def logit_processor_func(input_ids, logits): logits_processor=logit_processors ) assert output["choices"][0]["text"].lower().startswith("rot") + + model.set_seed(1337) + + state = model.save_state() + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_1 = output["choices"][0]["text"] + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_2 = output["choices"][0]["text"] + + model.load_state(state) + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_3 = output["choices"][0]["text"] + + assert number_1 != number_2 + assert number_1 == number_3 From 29afcfdff5e75d7df4c13bad0122c98661d251ab Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 23:04:59 -0400 Subject: [PATCH 076/155] fix: Don't store scores internally unless logits_all=True. Reduces memory requirements for large context. Closes #1542 --- llama_cpp/llama.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 80fadffe2..70623d813 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -451,7 +451,7 @@ def free_lora_adapter(): self.n_tokens = 0 self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) self.scores: npt.NDArray[np.single] = np.ndarray( - (n_ctx, self._n_vocab), dtype=np.single + (n_ctx if logits_all == True else n_batch, self._n_vocab), dtype=np.single ) self._mirostat_mu = ctypes.c_float( @@ -648,12 +648,14 @@ def eval(self, tokens: Sequence[int]): ) self.scores[n_past : n_past + n_tokens, :].reshape(-1)[::] = logits else: - rows = 1 - cols = self._n_vocab - logits = np.ctypeslib.as_array( - self._ctx.get_logits(), shape=(rows * cols,) - ) - self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits + # rows = 1 + # cols = self._n_vocab + # logits = np.ctypeslib.as_array( + # self._ctx.get_logits(), shape=(rows * cols,) + # ) + # self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits + # NOTE: Now that sampling is done inside the sampler, logits are only needed for logprobs which requires logits_all + pass # Update n_tokens self.n_tokens += n_tokens From 84c092063e8f222758dd3d60bdb2d1d342ac292e Mon Sep 17 00:00:00 2001 From: Jonathan Jordan Date: Thu, 19 Sep 2024 05:09:01 +0200 Subject: [PATCH 077/155] feat: Add loading sharded GGUF files from HuggingFace with Llama.from_pretrained(additional_files=[...]) . Closes #1341 Co-authored-by: Andrei --- llama_cpp/llama.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 70623d813..3505dcfa1 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2227,6 +2227,7 @@ def from_pretrained( cls, repo_id: str, filename: Optional[str], + additional_files: Optional[List] = None, local_dir: Optional[Union[str, os.PathLike[str]]] = None, local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", cache_dir: Optional[Union[str, os.PathLike[str]]] = None, @@ -2239,6 +2240,7 @@ def from_pretrained( Args: repo_id: The model repo id. filename: A filename or glob pattern to match the model file in the repo. + additional_files: A list of filenames or glob patterns to match additional model files in the repo. local_dir: The local directory to save the model to. local_dir_use_symlinks: Whether to use symlinks when downloading the model. **kwargs: Additional keyword arguments to pass to the Llama constructor. @@ -2269,6 +2271,7 @@ def from_pretrained( rel_path = Path(file).relative_to(repo_id) file_list.append(str(rel_path)) + # find the only/first shard file: matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore if len(matching_files) == 0: @@ -2298,6 +2301,35 @@ def from_pretrained( cache_dir=cache_dir, ) + if additional_files: + for additonal_file_name in additional_files: + # find the additional shard file: + matching_additional_files = [file for file in file_list if fnmatch.fnmatch(file, additonal_file_name)] + + if len(matching_additional_files) == 0: + raise ValueError( + f"No file found in {repo_id} that match {additonal_file_name}\n\n" + f"Available Files:\n{json.dumps(file_list)}" + ) + + if len(matching_additional_files) > 1: + raise ValueError( + f"Multiple files found in {repo_id} matching {additonal_file_name}\n\n" + f"Available Files:\n{json.dumps(files)}" + ) + + (matching_additional_file,) = matching_additional_files + + # download the additional file + hf_hub_download( + repo_id=repo_id, + filename=matching_additional_file, + subfolder=subfolder, + local_dir=local_dir, + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cache_dir, + ) + if local_dir is None: model_path = hf_hub_download( repo_id=repo_id, @@ -2311,6 +2343,7 @@ def from_pretrained( else: model_path = os.path.join(local_dir, filename) + # loading the first file of a sharded GGUF loads all remaining shard files in the subfolder return cls( model_path=model_path, **kwargs, From 47d7a621335d4f1bc7b96e4d9a39d663c32db22b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 20 Sep 2024 18:02:11 -0400 Subject: [PATCH 078/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 64c6af319..6026da52d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 64c6af3195c3cd4aa3328a1282d29cd2635c34c9 +Subproject commit 6026da52d6942b253df835070619775d849d0258 From 6c44a3f36b089239cb6396bb408116aad262c702 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 20 Sep 2024 18:03:37 -0400 Subject: [PATCH 079/155] feat: Add option to configure n_ubatch --- llama_cpp/llama.py | 5 +++++ llama_cpp/server/model.py | 1 + llama_cpp/server/settings.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3505dcfa1..ae712dedb 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -75,6 +75,7 @@ def __init__( seed: int = llama_cpp.LLAMA_DEFAULT_SEED, n_ctx: int = 512, n_batch: int = 512, + n_ubatch: int = 512, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, rope_scaling_type: Optional[ @@ -156,6 +157,7 @@ def __init__( seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_batch: Prompt processing maximum batch size + n_ubatch: Physical batch size n_threads: Number of threads to use for generation n_threads_batch: Number of threads to use for batch processing rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054 @@ -309,6 +311,7 @@ def __init__( self.context_params = llama_cpp.llama_context_default_params() self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch self.context_params.rope_scaling_type = ( @@ -380,6 +383,7 @@ def __init__( self.n_batch = min(n_ctx, n_batch) self.context_params.n_ctx = self._model.n_ctx_train() self.context_params.n_batch = self.n_batch + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) self._ctx = self._stack.enter_context( contextlib.closing( @@ -2071,6 +2075,7 @@ def __getstate__(self): seed=self.context_params.seed, n_ctx=self.context_params.n_ctx, n_batch=self.n_batch, + n_ubatch=self.context_params.n_ubatch, n_threads=self.context_params.n_threads, n_threads_batch=self.context_params.n_threads_batch, rope_scaling_type=self.context_params.rope_scaling_type, diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 901998094..c6716f919 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -249,6 +249,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: seed=settings.seed, n_ctx=settings.n_ctx, n_batch=settings.n_batch, + n_ubatch=settings.n_ubatch, n_threads=settings.n_threads, n_threads_batch=settings.n_threads_batch, rope_scaling_type=settings.rope_scaling_type, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index b20655813..13c951241 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -70,6 +70,9 @@ class ModelSettings(BaseSettings): n_batch: int = Field( default=512, ge=1, description="The batch size to use per eval." ) + n_ubatch: int = Field( + default=512, ge=1, description="The physical batch size used by llama.cpp" + ) n_threads: int = Field( default=max(multiprocessing.cpu_count() // 2, 1), ge=1, From 49b1e7379f5bb308acdba470e561f280008d8f15 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 21 Sep 2024 00:04:34 +0200 Subject: [PATCH 080/155] docs: Add cuda 12.5 to README.md (#1750) Typo fix Co-authored-by: Andrei --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12b4b127d..dbaec5077 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Where `` is one of the following: - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 - `cu124`: CUDA 12.4 -- `cu124`: CUDA 12.5 +- `cu125`: CUDA 12.5 For example, to install the CUDA 12.1 wheel: From 1324c0c50c53e7987ff2ad1ba6d2bbe3fb13af69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:04:59 -0400 Subject: [PATCH 081/155] chore(deps): bump actions/cache from 3 to 4 (#1751) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/test.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index b1e5fc1a9..647b171e8 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -72,7 +72,7 @@ jobs: - name: VS Integration Cache id: vs-integration-cache if: runner.os == 'Windows' - uses: actions/cache@v4.0.2 + uses: actions/cache@v4 with: path: ./MSBuildExtensions key: cuda-${{ matrix.cuda }}-vs-integration diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f24f5b7e1..8f619cf1f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,7 +46,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -86,7 +86,7 @@ jobs: cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -129,7 +129,7 @@ jobs: cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -168,7 +168,7 @@ jobs: python-version: "3.9" - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} From 4744551b3b777d9889ee0fdd25409101d57d59f5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 21 Sep 2024 20:53:16 -0400 Subject: [PATCH 082/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6026da52d..d09770cae 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6026da52d6942b253df835070619775d849d0258 +Subproject commit d09770cae71b416c032ec143dda530f7413c4038 From 926b4143e865bc6ffd405a952639099214385139 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 13:23:18 -0400 Subject: [PATCH 083/155] feat: Update llama.cpp --- llama_cpp/_logger.py | 7 +++++++ vendor/llama.cpp | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 157af692f..787b3f108 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -10,6 +10,7 @@ # GGML_LOG_LEVEL_WARN = 2, # GGML_LOG_LEVEL_ERROR = 3, # GGML_LOG_LEVEL_DEBUG = 4, +# GGML_LOG_LEVEL_CONT = 5, // continue previous log # }; GGML_LOG_LEVEL_TO_LOGGING_LEVEL = { 0: logging.CRITICAL, @@ -17,10 +18,12 @@ 2: logging.WARNING, 3: logging.ERROR, 4: logging.DEBUG, + 5: logging.DEBUG, } logger = logging.getLogger("llama-cpp-python") +_last_log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[0] # typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); @llama_cpp.llama_log_callback @@ -29,8 +32,12 @@ def llama_log_callback( text: bytes, user_data: ctypes.c_void_p, ): + # TODO: Correctly implement continue previous log + global _last_log_level + log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level if logger.level <= GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level]: print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) + _last_log_level = log_level llama_cpp.llama_log_set(llama_log_callback, ctypes.c_void_p(0)) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d09770cae..ea9c32be7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d09770cae71b416c032ec143dda530f7413c4038 +Subproject commit ea9c32be71b91b42ecc538bd902e93cbb5fb36cb From b3dfb420e79fcdb326bc3bac861f73a152380946 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:05:08 -0400 Subject: [PATCH 084/155] chore: Bump version --- CHANGELOG.md | 13 +++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f01b1b9bf..14f68a456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ea9c32be71b91b42ecc538bd902e93cbb5fb36cb +- feat: Enable detokenizing special tokens with special=True by @benniekiss in #1596 +- feat(ci): Speed up CI workflows using uv, add support for CUDA 12.5 wheels by @Smartappli in e529940f45d42ed8aa31334123b8d66bc67b0e78 +- feat: Add loading sharded GGUF files from HuggingFace with Llama.from_pretrained(additional_files=[...]) by @Gnurro in 84c092063e8f222758dd3d60bdb2d1d342ac292e +- feat: Add option to configure n_ubatch by @abetlen in 6c44a3f36b089239cb6396bb408116aad262c702 +- feat: Update sampling API for llama.cpp. Sampling now uses sampler chain by @abetlen in f8fcb3ea3424bcfba3a5437626a994771a02324b +- fix: Don't store scores internally unless logits_all=True. Reduces memory requirements for large context by @abetlen in 29afcfdff5e75d7df4c13bad0122c98661d251ab +- fix: Fix memory allocation of ndarray in by @xu-song in #1704 +- fix: Use system message in og qwen format by @abetlen in 98eb092d3c6e7c142c4ba2faaca6c091718abbb3 + + ## [0.2.90] - feat: Update llama.cpp to ggerganov/llama.cpp@1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index db1f1d0fa..70fde2041 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.90" +__version__ = "0.3.0" From 8e07db0f87f2e0cd543f1449a5139b7026a19e9f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:17:04 -0400 Subject: [PATCH 085/155] fix: install build dependency --- .github/workflows/publish.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 67816ea03..bb76f5394 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,34 +13,36 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" - + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.9" - + - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' env: - RUST_LOG: trace + RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: cmd - + - name: Build source distribution run: | python -m build --sdist - + - name: Publish distribution to PyPI # TODO: move to tag based releases # if: startsWith(github.ref, 'refs/tags') From 65222bc4931e8d5cf637a8d5f966da2f46897837 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:27:19 -0400 Subject: [PATCH 086/155] fix: install build dependency --- .github/workflows/build-and-release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 3f85a30fe..bfa97decd 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" - + - uses: actions/setup-python@v5 with: python-version: "3.9" @@ -103,6 +103,7 @@ jobs: python -m pip install --upgrade pip python -m pip install uv RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: bash - name: Install dependencies (Windows) @@ -113,12 +114,13 @@ jobs: python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: cmd - + - name: Build source distribution run: | python -m build --sdist - + - uses: actions/upload-artifact@v4 with: name: sdist From 9992c5084a3df2f533e265d10f81d4269b97a1e6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 00:34:21 -0400 Subject: [PATCH 087/155] fix: Fix speculative decoding --- llama_cpp/llama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ae712dedb..2ae65a83d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -807,8 +807,10 @@ def sample( grammar=grammar, ) + ridx = idx - self.n_tokens if idx is not None else -1 + assert self.ctx is not None - token = self._sampler.sample(self._ctx, -1) + token = self._sampler.sample(self._ctx, ridx) if tmp_sampler: self._sampler = None return token From 11d9562fd63473dbf4ec91d942ca5f197deec462 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Thu, 26 Sep 2024 12:37:17 +0800 Subject: [PATCH 088/155] misc: Rename all_text to remaining_text (#1658) Co-authored-by: Andrei --- llama_cpp/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2ae65a83d..4f22fae84 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1519,15 +1519,15 @@ def logit_bias_processor( if stream: remaining_tokens = completion_tokens[returned_tokens:] - all_text = self.detokenize( + remaining_text = self.detokenize( remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens], ) - any_stop = [s for s in stop_sequences if s in all_text] + any_stop = [s for s in stop_sequences if s in remaining_text] if len(any_stop) > 0: - end = min(all_text.index(stop) for stop in any_stop) + end = min(remaining_text.index(stop) for stop in any_stop) else: - end = len(all_text) + end = len(remaining_text) token_end_position = 0 for token in remaining_tokens: From e975dabf74b3ad85689c9a07719cbb181313139b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 01:11:02 -0400 Subject: [PATCH 089/155] fix: Additional fixes for speculative decoding --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4f22fae84..babb30cf0 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -930,7 +930,7 @@ def generate( sample_idx += 1 if stopping_criteria is not None and stopping_criteria( - self._input_ids, self._scores[-1, :] + self._input_ids[: sample_idx], self._scores[sample_idx - self.n_tokens, :] ): return tokens_or_none = yield token From dca0c9a09cea39a8454dd897253be540e82f369d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 10:09:22 -0400 Subject: [PATCH 090/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ea9c32be7..95bc82fbc 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ea9c32be71b91b42ecc538bd902e93cbb5fb36cb +Subproject commit 95bc82fbc0df6d48cf66c857a4dda3d044f45ca2 From 01c76078eea77f12fb01aff605859afc372bf598 Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 26 Sep 2024 11:23:00 -0400 Subject: [PATCH 091/155] feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava --- llama_cpp/_ctypes_extensions.py | 131 ++++++++++++++++++++++++++++++ llama_cpp/_ggml.py | 12 +++ llama_cpp/llama_cpp.py | 140 ++++---------------------------- llama_cpp/llava_cpp.py | 113 +++----------------------- 4 files changed, 171 insertions(+), 225 deletions(-) create mode 100644 llama_cpp/_ctypes_extensions.py create mode 100644 llama_cpp/_ggml.py diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py new file mode 100644 index 000000000..e88ed387d --- /dev/null +++ b/llama_cpp/_ctypes_extensions.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import sys +import os +import ctypes +import functools +import pathlib + +from typing import ( + Any, + Callable, + List, + Union, + Optional, + TYPE_CHECKING, + TypeVar, + Generic, +) +from typing_extensions import TypeAlias + + +# Load the library +def load_shared_library(lib_base_name: str, base_path: pathlib.Path): + """Platform independent shared library loader""" + # Searching for the library in the current directory under the name "libllama" (default name + # for llamacpp) and "llama" (default name for this repo) + lib_paths: List[pathlib.Path] = [] + # Determine the file extension based on the platform + if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): + lib_paths += [ + base_path / f"lib{lib_base_name}.so", + ] + elif sys.platform == "darwin": + lib_paths += [ + base_path / f"lib{lib_base_name}.so", + base_path / f"lib{lib_base_name}.dylib", + ] + elif sys.platform == "win32": + lib_paths += [ + base_path / f"{lib_base_name}.dll", + base_path / f"lib{lib_base_name}.dll", + ] + else: + raise RuntimeError("Unsupported platform") + + cdll_args = dict() # type: ignore + + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32": + os.add_dll_directory(str(base_path)) + os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"] + + if sys.platform == "win32" and sys.version_info >= (3, 8): + os.add_dll_directory(str(base_path)) + if "CUDA_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) + if "HIP_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + cdll_args["winmode"] = ctypes.RTLD_GLOBAL + + # Try to load the shared library, handling potential errors + for lib_path in lib_paths: + if lib_path.exists(): + try: + return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore + except Exception as e: + raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") + + raise FileNotFoundError( + f"Shared library with base name '{lib_base_name}' not found" + ) + + +# ctypes sane type hint helpers +# +# - Generic Pointer and Array types +# - PointerOrRef type with a type hinted byref function +# +# NOTE: Only use these for static type checking not for runtime checks +# no good will come of that + +if TYPE_CHECKING: + CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore + + CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore + + CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore + + CtypesVoidPointer: TypeAlias = ctypes.c_void_p + + class CtypesRef(Generic[CtypesCData]): + pass + + CtypesPointerOrRef: TypeAlias = Union[ + CtypesPointer[CtypesCData], CtypesRef[CtypesCData] + ] + + CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore + +F = TypeVar("F", bound=Callable[..., Any]) + + +def ctypes_function_for_shared_library(lib: ctypes.CDLL): + """Decorator for defining ctypes functions with type hints""" + + def ctypes_function( + name: str, argtypes: List[Any], restype: Any, enabled: bool = True + ): + def decorator(f: F) -> F: + if enabled: + func = getattr(lib, name) + func.argtypes = argtypes + func.restype = restype + functools.wraps(f)(func) + return func + else: + return f + + return decorator + + return ctypes_function + + +def _byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCData]: + """Type-annotated version of ctypes.byref""" + ... + + +byref = _byref if TYPE_CHECKING else ctypes.byref diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py new file mode 100644 index 000000000..5bee8a93b --- /dev/null +++ b/llama_cpp/_ggml.py @@ -0,0 +1,12 @@ +"""Internal module use at your own risk + +This module provides a minimal interface for working with ggml tensors from llama-cpp-python +""" +import os +import pathlib + +import llama_cpp._ctypes_extensions as ctypes_ext + +libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" +libggml = ctypes_ext.load_shared_library("ggml", libggml_base_path) + diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6b82753e1..de427970c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1,151 +1,45 @@ from __future__ import annotations -import sys import os import ctypes -import functools import pathlib from typing import ( - Any, Callable, - List, Union, NewType, Optional, TYPE_CHECKING, - TypeVar, - Generic, ) -from typing_extensions import TypeAlias +from llama_cpp._ctypes_extensions import ( + load_shared_library, + byref, + ctypes_function_for_shared_library, +) -# Load the library -def _load_shared_library(lib_base_name: str): - # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" - # Searching for the library in the current directory under the name "libllama" (default name - # for llamacpp) and "llama" (default name for this repo) - _lib_paths: List[pathlib.Path] = [] - # Determine the file extension based on the platform - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - ] - elif sys.platform == "darwin": - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - _base_path / f"lib{lib_base_name}.dylib", - ] - elif sys.platform == "win32": - _lib_paths += [ - _base_path / f"{lib_base_name}.dll", - _base_path / f"lib{lib_base_name}.dll", - ] - else: - raise RuntimeError("Unsupported platform") - - if "LLAMA_CPP_LIB" in os.environ: - lib_base_name = os.environ["LLAMA_CPP_LIB"] - _lib = pathlib.Path(lib_base_name) - _base_path = _lib.parent.resolve() - _lib_paths = [_lib.resolve()] - - cdll_args = dict() # type: ignore - - # Add the library directory to the DLL search path on Windows (if needed) - if sys.platform == "win32": - os.add_dll_directory(str(_base_path)) - os.environ["PATH"] = str(_base_path) + os.pathsep + os.environ["PATH"] - - if sys.platform == "win32" and sys.version_info >= (3, 8): - os.add_dll_directory(str(_base_path)) - if "CUDA_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) - if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) - cdll_args["winmode"] = ctypes.RTLD_GLOBAL - - # Try to load the shared library, handling potential errors - for _lib_path in _lib_paths: - if _lib_path.exists(): - try: - return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore - except Exception as e: - raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}") - - raise FileNotFoundError( - f"Shared library with base name '{lib_base_name}' not found" +if TYPE_CHECKING: + from llama_cpp._ctypes_extensions import ( + CtypesCData, + CtypesArray, + CtypesPointer, + CtypesVoidPointer, + CtypesRef, + CtypesPointerOrRef, + CtypesFuncPointer, ) # Specify the base name of the shared library to load _lib_base_name = "llama" - +_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH") +_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path) # Load the library -_lib = _load_shared_library(_lib_base_name) - - -# ctypes sane type hint helpers -# -# - Generic Pointer and Array types -# - PointerOrRef type with a type hinted byref function -# -# NOTE: Only use these for static type checking not for runtime checks -# no good will come of that - -if TYPE_CHECKING: - CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore - - CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore - - CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore - - CtypesVoidPointer: TypeAlias = ctypes.c_void_p - - class CtypesRef(Generic[CtypesCData]): - pass - - CtypesPointerOrRef: TypeAlias = Union[ - CtypesPointer[CtypesCData], CtypesRef[CtypesCData] - ] - - CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore - -F = TypeVar("F", bound=Callable[..., Any]) - - -def ctypes_function_for_shared_library(lib: ctypes.CDLL): - def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True - ): - def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) - func.argtypes = argtypes - func.restype = restype - functools.wraps(f)(func) - return func - else: - return f - - return decorator - - return ctypes_function - +_lib = load_shared_library(_lib_base_name, _base_path) ctypes_function = ctypes_function_for_shared_library(_lib) -def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCData]: - """Type-annotated version of ctypes.byref""" - ... - - -byref = ctypes.byref # type: ignore - # from ggml.h # // NOTE: always add types at the end of the enum to keep backward compatibility # enum ggml_type { diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index 7d97dc0fd..d9dfaf5fd 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -1,9 +1,6 @@ from __future__ import annotations -import sys import os -import ctypes -import functools from ctypes import ( c_bool, c_char_p, @@ -17,121 +14,32 @@ ) import pathlib from typing import ( - List, Union, NewType, Optional, - TypeVar, - Callable, - Any, TYPE_CHECKING, - Generic, ) -from typing_extensions import TypeAlias import llama_cpp.llama_cpp as llama_cpp +from llama_cpp._ctypes_extensions import ( + load_shared_library, + ctypes_function_for_shared_library, +) -# Load the library -def _load_shared_library(lib_base_name: str): - # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" - # Searching for the library in the current directory under the name "libllama" (default name - # for llamacpp) and "llama" (default name for this repo) - _lib_paths: List[pathlib.Path] = [] - # Determine the file extension based on the platform - if sys.platform.startswith("linux"): - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - ] - elif sys.platform == "darwin": - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - _base_path / f"lib{lib_base_name}.dylib", - ] - elif sys.platform == "win32": - _lib_paths += [ - _base_path / f"{lib_base_name}.dll", - _base_path / f"lib{lib_base_name}.dll", - ] - else: - raise RuntimeError("Unsupported platform") - - if "LLAVA_CPP_LIB" in os.environ: - lib_base_name = os.environ["LLAVA_CPP_LIB"] - _lib = pathlib.Path(lib_base_name) - _base_path = _lib.parent.resolve() - _lib_paths = [_lib.resolve()] - - cdll_args = dict() # type: ignore - # Add the library directory to the DLL search path on Windows (if needed) - if sys.platform == "win32" and sys.version_info >= (3, 8): - os.add_dll_directory(str(_base_path)) - if "CUDA_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) - cdll_args["winmode"] = ctypes.RTLD_GLOBAL - - # Try to load the shared library, handling potential errors - for _lib_path in _lib_paths: - if _lib_path.exists(): - try: - return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore - except Exception as e: - raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}") - - raise FileNotFoundError( - f"Shared library with base name '{lib_base_name}' not found" +if TYPE_CHECKING: + from llama_cpp._ctypes_extensions import ( + CtypesArray, ) # Specify the base name of the shared library to load _libllava_base_name = "llava" +_libllava_override_path = os.environ.get("LLAVA_CPP_LIB") +_libllava_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libllava_override_path is None else pathlib.Path() # Load the library -_libllava = _load_shared_library(_libllava_base_name) - -# ctypes helper - -if TYPE_CHECKING: - CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore - - CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore - - CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore - - CtypesVoidPointer: TypeAlias = ctypes.c_void_p - - class CtypesRef(Generic[CtypesCData]): - pass - - CtypesPointerOrRef: TypeAlias = Union[ - CtypesPointer[CtypesCData], CtypesRef[CtypesCData] - ] - - CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore - -F = TypeVar("F", bound=Callable[..., Any]) - - -def ctypes_function_for_shared_library(lib: ctypes.CDLL): - def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True - ): - def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) - func.argtypes = argtypes - func.restype = restype - functools.wraps(f)(func) - return func - else: - return f - - return decorator - - return ctypes_function - +_libllava = load_shared_library(_libllava_base_name, _libllava_base_path) ctypes_function = ctypes_function_for_shared_library(_libllava) @@ -247,3 +155,4 @@ def clip_model_load( @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) def clip_free(ctx: clip_ctx_p, /): ... + From 57e70bb5693fb0330b11c50f707adb88a6be7b85 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 29 Sep 2024 15:23:24 -0400 Subject: [PATCH 092/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 16 ++++++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index de427970c..97c969136 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -220,6 +220,7 @@ # LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -247,6 +248,7 @@ LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 +LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 # // note: these values should be synchronized with ggml_rope @@ -404,12 +406,14 @@ # LLAMA_POOLING_TYPE_MEAN = 1, # LLAMA_POOLING_TYPE_CLS = 2, # LLAMA_POOLING_TYPE_LAST = 3, +# LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph # }; LLAMA_POOLING_TYPE_UNSPECIFIED = -1 LLAMA_POOLING_TYPE_NONE = 0 LLAMA_POOLING_TYPE_MEAN = 1 LLAMA_POOLING_TYPE_CLS = 2 LLAMA_POOLING_TYPE_LAST = 3 +LLAMA_POOLING_TYPE_RANK = 4 # enum llama_attention_type { # LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, @@ -420,10 +424,11 @@ LLAMA_ATTENTION_TYPE_CAUSAL = 0 LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 + # enum llama_split_mode { -# LLAMA_SPLIT_MODE_NONE = 0, // single GPU -# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs -# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs +# LLAMA_SPLIT_MODE_NONE = 0, // single GPU +# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs +# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs # }; LLAMA_SPLIT_MODE_NONE = 0 LLAMA_SPLIT_MODE_LAYER = 1 @@ -2520,7 +2525,8 @@ def llama_get_embeddings_ith( # // Get the embeddings for a sequence id # // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE -# // shape: [n_embd] (1-dimensional) +# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence +# // otherwise: float[n_embd] (1-dimensional) # LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); @ctypes_function( "llama_get_embeddings_seq", @@ -2672,6 +2678,8 @@ def llama_token_eot(model: llama_model_p, /) -> int: # // # // Tokenization # // +# // The API is thread-safe. +# // # /// @details Convert the provided text into tokens. diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 95bc82fbc..c919d5db3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 95bc82fbc0df6d48cf66c857a4dda3d044f45ca2 +Subproject commit c919d5db39c8a7fcb64737f008e4b105ee0acd20 From 7c4aead82d349469bbbe7d8c0f4678825873c039 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 29 Sep 2024 15:26:54 -0400 Subject: [PATCH 093/155] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f68a456..c68e96b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.1] + +- feat: Update llama.cpp to ggerganov/llama.cpp@c919d5db39c8a7fcb64737f008e4b105ee0acd20 +- feat: Expose libggml in internal APIs by @abetlen in #1761 +- fix: Fix speculative decoding by @abetlen in 9992c5084a3df2f533e265d10f81d4269b97a1e6 and e975dabf74b3ad85689c9a07719cbb181313139b +- misc: Rename all_text to remaining_text by @xu-song in #1658 + ## [0.3.0] - feat: Update llama.cpp to ggerganov/llama.cpp@ea9c32be71b91b42ecc538bd902e93cbb5fb36cb diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 70fde2041..c11b879d6 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.0" +__version__ = "0.3.1" From 7403e002b8e033c0a34e93fba2b311e2118487fe Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 22 Oct 2024 09:33:48 -0400 Subject: [PATCH 094/155] feat: Update llama.cpp --- CMakeLists.txt | 18 ++++- llama_cpp/llama_cpp.py | 152 ++++++++++++++++++++++++++++++++--------- vendor/llama.cpp | 2 +- 3 files changed, 139 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6b35ed6c..9af5b2180 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,9 @@ if (LLAMA_BUILD) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set(CMAKE_SKIP_RPATH FALSE) + # Enable building of the common library + set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) + # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") # Need to disable these llama.cpp flags on Apple x86_64, @@ -106,7 +109,7 @@ if (LLAMA_BUILD) # Building llava add_subdirectory(vendor/llama.cpp/examples/llava) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - # Set CUDA_ARCHITECTURES to OFF on windows + # Set CUDA_ARCHITECTURES to OFF on Windows if (WIN32) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() @@ -121,5 +124,18 @@ if (LLAMA_BUILD) DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib ) endif() + + # Fix for llava build: Add include directory for llama.h + # Move these commands after the add_subdirectory call + target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + + if (BUILD_SHARED_LIBS) + target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + endif() + + target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) endif() endif() diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 97c969136..66feed896 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -464,6 +464,8 @@ class llama_token_data(ctypes.Structure): # typedef struct llama_token_data_array { +# // TODO: consider SoA +# // NOTE: this pointer can be modified by the samplers # llama_token_data * data; # size_t size; # int64_t selected; // this is the index in the data array (i.e. not the token id) @@ -507,8 +509,11 @@ class llama_token_data_array(ctypes.Structure): # // - token : the token ids of the input (used when embd is NULL) # // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) # // - pos : the positions of the respective token in the sequence +# // (if set to NULL, the token position will be tracked automatically by llama_decode) # // - seq_id : the sequence to which the respective token belongs +# // (if set to NULL, the sequence ID will be assumed to be 0) # // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output +# // (if set to NULL, only the logits for last token will be returned) # // # typedef struct llama_batch { # int32_t n_tokens; @@ -519,16 +524,6 @@ class llama_token_data_array(ctypes.Structure): # int32_t * n_seq_id; # llama_seq_id ** seq_id; # int8_t * logits; // TODO: rename this to "output" - - -# // NOTE: helpers for smooth API transition - can be deprecated in the future -# // for future-proof code, use the above fields instead and ignore everything below -# // -# // pos[i] = all_pos_0 + i*all_pos_1 -# // -# llama_pos all_pos_0; // used if pos == NULL -# llama_pos all_pos_1; // used if pos == NULL -# llama_seq_id all_seq_id; // used if seq_id == NULL # } llama_batch; class llama_batch(ctypes.Structure): """Input data for llama_decode @@ -563,9 +558,6 @@ class llama_batch(ctypes.Structure): ("n_seq_id", ctypes.POINTER(ctypes.c_int32)), ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))), ("logits", ctypes.POINTER(ctypes.c_int8)), - ("all_pos_0", llama_pos), - ("all_pos_1", llama_pos), - ("all_seq_id", llama_seq_id), ] @@ -1170,6 +1162,12 @@ def llama_supports_gpu_offload() -> bool: ... +# LLAMA_API bool llama_supports_rpc (void); +@ctypes_function("llama_supports_rpc", [], ctypes.c_bool) +def llama_supports_rpc() -> bool: + ... + + # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) def llama_n_ctx(ctx: llama_context_p, /) -> int: @@ -2255,30 +2253,26 @@ def llama_state_seq_load_file( # // -# // Return batch for single sequence of tokens starting at pos_0 +# // Return batch for single sequence of tokens +# // The sequence ID will be fixed to 0 +# // The position of the tokens will be tracked automatically by llama_decode # // # // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it # // # LLAMA_API struct llama_batch llama_batch_get_one( # llama_token * tokens, -# int32_t n_tokens, -# llama_pos pos_0, -# llama_seq_id seq_id); +# int32_t n_tokens); @ctypes_function( "llama_batch_get_one", [ llama_token_p, - ctypes.c_int, - llama_pos, - llama_seq_id, + ctypes.c_int32, ], llama_batch, ) def llama_batch_get_one( tokens: CtypesArray[llama_token], n_tokens: Union[ctypes.c_int, int], - pos_0: Union[llama_pos, int], - seq_id: llama_seq_id, /, ) -> llama_batch: """Return batch for single sequence of tokens starting at pos_0 @@ -2616,6 +2610,13 @@ def llama_token_eos(model: llama_model_p, /) -> int: ... +# LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn +@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) +def llama_token_eot(model: llama_model_p, /) -> int: + """end-of-turn""" + ... + + # LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification @ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token) def llama_token_cls(model: llama_model_p, /) -> int: @@ -2650,30 +2651,54 @@ def llama_add_eos_token(model: llama_model_p, /) -> bool: # // Codellama infill tokens -# LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix +# DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead"); @ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token) def llama_token_prefix(model: llama_model_p) -> int: """codellama infill tokens""" ... -# LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle +# DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead"); @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) def llama_token_middle(model: llama_model_p, /) -> int: ... -# LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix +# DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead"); @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) def llama_token_suffix(model: llama_model_p, /) -> int: ... -# LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle -@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model); +@ctypes_function("llama_token_fim_pre", [llama_model_p_ctypes], llama_token) +def llama_token_fim_pre(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model); +@ctypes_function("llama_token_fim_suf", [llama_model_p_ctypes], llama_token) +def llama_token_fim_suf(model: llama_model_p, /) -> int: ... +# LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model); +@ctypes_function("llama_token_fim_mid", [llama_model_p_ctypes], llama_token) +def llama_token_fim_mid(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model); +@ctypes_function("llama_token_fim_pad", [llama_model_p_ctypes], llama_token) +def llama_token_fim_pad(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model); +@ctypes_function("llama_token_fim_rep", [llama_model_p_ctypes], llama_token) +def llama_token_fim_rep(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model); +@ctypes_function("llama_token_fim_sep", [llama_model_p_ctypes], llama_token) +def llama_token_fim_sep(model: llama_model_p, /) -> int: + ... # // # // Tokenization @@ -2786,6 +2811,23 @@ def llama_token_to_piece( ... +# # // check if token0 is contained as a prefix in token1 +# # LLAMA_API bool llama_token_is_prefix( +# # const struct llama_model * model, +# # llama_token token0, +# # llama_token token1); +# @ctypes_function( +# "llama_token_is_prefix", +# [llama_model_p_ctypes, llama_token, llama_token], +# ctypes.c_bool, +# ) +# def llama_token_is_prefix( +# model: llama_model_p, token0: Union[llama_token, int], token1: Union[llama_token, int], / +# ) -> bool: +# """Check if token0 is contained as a prefix in token1""" +# ... + + # /// @details Convert the provided tokens into text (inverse of llama_tokenize()). # /// @param text The char pointer must be large enough to hold the resulting text. # /// @return Returns the number of chars/bytes on success, no more than text_len_max. @@ -3099,20 +3141,22 @@ def llama_sampler_chain_remove( # // available samplers: # -# LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); +# LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) def llama_sampler_init_greedy() -> llama_sampler_p: ... -# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); +# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); @ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. -# LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); +# /// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first. +# DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void), +# "will be removed in the future (see https://github.com/ggerganov/llama.cpp/pull/9896#discussion_r1800920915)"); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) def llama_sampler_init_softmax() -> llama_sampler_p: ... @@ -3188,6 +3232,19 @@ def llama_sampler_init_temp_ext( ... +# /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 +# LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); +@ctypes_function( + "llama_sampler_init_xtc", + [ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32], + llama_sampler_p_ctypes, +) +def llama_sampler_init_xtc( + p: float, t: float, min_keep: int, seed: int, / +) -> llama_sampler_p: + ... + + # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3301,6 +3358,39 @@ def llama_sampler_init_logit_bias( ... +# // this sampler is meant to be used for fill-in-the-middle infilling +# // it's supposed to be used after top_k + top_p sampling +# // +# // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG +# // 2. combine probs of tokens that have the same prefix +# // +# // example: +# // +# // - before: +# // "hel": 0.5 +# // "hell": 0.2 +# // "hello": 0.1 +# // "dummy": 0.1 +# // +# // - after: +# // "hel": 0.8 +# // "dummy": 0.1 +# // +# // 3. discard non-EOG tokens with low prob +# // 4. if no tokens are left -> pick EOT +# // +# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_model * model); +@ctypes_function( + "llama_sampler_init_infill", + [llama_model_p_ctypes], + llama_sampler_p_ctypes, +) +def llama_sampler_init_infill(model: llama_model_p, /) -> llama_sampler_p: + """This sampler is meant to be used for fill-in-the-middle infilling. + """ + ... + + # // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise # LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); @ctypes_function( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c919d5db3..c421ac072 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c919d5db39c8a7fcb64737f008e4b105ee0acd20 +Subproject commit c421ac072d46172ab18924e1e8be53680b54ed3b From e712cffc0c22d5406c6fe02a1769992f2181f229 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 31 Oct 2024 13:50:18 -0400 Subject: [PATCH 095/155] feat: Update llama.cpp --- llama_cpp/_internals.py | 14 ------------- llama_cpp/llama.py | 1 - llama_cpp/llama_cpp.py | 46 +++++++++++++++++++++++++++++++---------- vendor/llama.cpp | 2 +- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 0aff34844..994d5f149 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -362,13 +362,6 @@ def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: i self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) - def sample_tail_free( - self, candidates: "_LlamaTokenDataArray", z: float, min_keep: int - ): - llama_cpp.llama_sample_tail_free( - self.ctx, llama_cpp.byref(candidates.candidates), z, min_keep - ) - def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): @@ -685,9 +678,6 @@ def sample( ctx_main.sample_top_k( token_data_array, self.params.top_k, min_keep=min_keep ) - ctx_main.sample_tail_free( - token_data_array, self.params.tfs_z, min_keep=min_keep - ) ctx_main.sample_typical( token_data_array, self.params.typical_p, min_keep=min_keep ) @@ -776,10 +766,6 @@ def add_min_p(self, p: float, min_keep: int): sampler = llama_cpp.llama_sampler_init_min_p(p, min_keep) self._add_sampler(sampler) - def add_tail_free(self, z: float, min_keep: int): - sampler = llama_cpp.llama_sampler_init_tail_free(z, min_keep) - self._add_sampler(sampler) - def add_typical(self, p: float, min_keep: int): sampler = llama_cpp.llama_sampler_init_typical(p, min_keep) self._add_sampler(sampler) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index babb30cf0..d15a88b00 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -745,7 +745,6 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): n_probs = 0 min_keep = max(1, n_probs) sampler.add_top_k(top_k) - sampler.add_tail_free(tfs_z, min_keep) sampler.add_typical(typical_p, min_keep) sampler.add_top_p(top_p, min_keep) sampler.add_min_p(min_p, min_keep) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 66feed896..457c6dddb 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3191,17 +3191,6 @@ def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... -# /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. -# LLAMA_API struct llama_sampler * llama_sampler_init_tail_free (float z, size_t min_keep); -@ctypes_function( - "llama_sampler_init_tail_free", - [ctypes.c_float, ctypes.c_size_t], - llama_sampler_p_ctypes, -) -def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: - ... - - # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. # LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); @ctypes_function( @@ -3343,6 +3332,41 @@ def llama_sampler_init_penalties( ... +# /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 +# LLAMA_API struct llama_sampler * llama_sampler_init_dry( +# const struct llama_model * model, +# float dry_multiplier, +# float dry_base, +# int32_t dry_allowed_length, +# int32_t dry_penalty_last_n, +# const char ** seq_breakers, +# size_t num_breakers); +@ctypes_function( + "llama_sampler_init_dry", + [ + llama_model_p_ctypes, + ctypes.c_float, + ctypes.c_float, + ctypes.c_int32, + ctypes.c_int32, + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ], + llama_sampler_p_ctypes, +) +def llama_sampler_init_dry( + model: llama_model_p, + dry_multiplier: float, + dry_base: float, + dry_allowed_length: int, + dry_penalty_last_n: int, + seq_breakers: CtypesArray[bytes], + num_breakers: int, + /, +) -> llama_sampler_p: + ... + + # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( # int32_t n_vocab, # int32_t n_logit_bias, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c421ac072..0a683e808 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c421ac072d46172ab18924e1e8be53680b54ed3b +Subproject commit 0a683e8088d849626e7471f9e2ed381f7dbdf2e9 From cafa33e1a8646258f2e44dd95782f4903d6944c0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 18:00:30 -0500 Subject: [PATCH 096/155] feat: Update llama.cpp --- CMakeLists.txt | 16 +++++++++++++++- vendor/llama.cpp | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9af5b2180..7d31ffd6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,21 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) - + llama_cpp_python_install_target(ggml-cpu) + llama_cpp_python_install_target(ggml-base) + + if (GGML_METAL) + llama_cpp_python_install_target(ggml-metal) + endif() + + if (GGML_CUDA) + llama_cpp_python_install_target(ggml-cuda) + endif() + + if (GGML_VULKAN) + llama_cpp_python_install_target(ggml-vulkan) + endif() + # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32) install( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 0a683e808..74d73dc85 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 0a683e8088d849626e7471f9e2ed381f7dbdf2e9 +Subproject commit 74d73dc85cc2057446bf63cc37ff649ae7cebd80 From d1cb50b49fafad171bca623f346cbc5438376fb3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:26:16 -0500 Subject: [PATCH 097/155] Add missing ggml dependency --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d31ffd6c..f8c55d339 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,7 @@ if (LLAMA_BUILD) llama_cpp_python_install_target(ggml) llama_cpp_python_install_target(ggml-cpu) llama_cpp_python_install_target(ggml-base) + llama_cpp_python_install_target(ggml-amx) if (GGML_METAL) llama_cpp_python_install_target(ggml-metal) From 2796f4e62f810780d89a3485a0a122a9e014e8ae Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:31:12 -0500 Subject: [PATCH 098/155] Add all missing ggml dependencies --- CMakeLists.txt | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8c55d339..c9f3c2b36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,10 @@ option(LLAMA_BUILD "Build llama.cpp shared library and install alongside python option(LLAVA_BUILD "Build llava shared library and install alongside python package" ON) function(llama_cpp_python_install_target target) + if(NOT TARGET ${target}) + return() + endif() + install( TARGETS ${target} LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib @@ -75,21 +79,21 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) - llama_cpp_python_install_target(ggml-cpu) - llama_cpp_python_install_target(ggml-base) - llama_cpp_python_install_target(ggml-amx) - - if (GGML_METAL) - llama_cpp_python_install_target(ggml-metal) - endif() - if (GGML_CUDA) - llama_cpp_python_install_target(ggml-cuda) - endif() + llama_cpp_python_install_target(ggml-base) - if (GGML_VULKAN) - llama_cpp_python_install_target(ggml-vulkan) - endif() + llama_cpp_python_install_target(ggml-amx) + llama_cpp_python_install_target(ggml-blas) + llama_cpp_python_install_target(ggml-can) + llama_cpp_python_install_target(ggml-cpu) + llama_cpp_python_install_target(ggml-cuda) + llama_cpp_python_install_target(ggml-hip) + llama_cpp_python_install_target(ggml-kompute) + llama_cpp_python_install_target(ggml-metal) + llama_cpp_python_install_target(ggml-musa) + llama_cpp_python_install_target(ggml-rpc) + llama_cpp_python_install_target(ggml-sycl) + llama_cpp_python_install_target(ggml-vulkan) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32) From 7ecdd944624cbd49e4af0a5ce1aa402607d58dcc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:55:09 -0500 Subject: [PATCH 099/155] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c68e96b87..8a194fece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.2] + +- feat: Update llama.cpp to ggerganov/llama.cpp@74d73dc85cc2057446bf63cc37ff649ae7cebd80 + ## [0.3.1] - feat: Update llama.cpp to ggerganov/llama.cpp@c919d5db39c8a7fcb64737f008e4b105ee0acd20 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c11b879d6..cda38a841 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.1" +__version__ = "0.3.2" From f3fb90b114835cc50c4816787d56bac2fe1180c3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 28 Nov 2024 18:27:55 -0500 Subject: [PATCH 100/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 22 ++++++++++++---------- vendor/llama.cpp | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 457c6dddb..bb2ba1993 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -615,6 +615,9 @@ class llama_model_kv_override(ctypes.Structure): # struct llama_model_params { +# // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) +# ggml_backend_dev_t * devices; + # int32_t n_gpu_layers; // number of layers to store in VRAM # enum llama_split_mode split_mode; // how to split the model across multiple GPUs @@ -680,6 +683,7 @@ class llama_model_params(ctypes.Structure): check_tensors: bool _fields_ = [ + ("devices", ctypes.c_void_p), # NOTE: unnused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), @@ -1898,6 +1902,14 @@ def llama_kv_cache_update(ctx: llama_context_p, /): ... +# // Check if the context supports KV cache shifting +# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); +@ctypes_function("llama_kv_cache_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: + """Check if the context supports KV cache shifting""" + ... + + # // # // State / sessions # // @@ -3621,13 +3633,3 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); -@ctypes_function( - "llama_perf_dump_yaml", - [ctypes.POINTER(ctypes.c_void_p), llama_context_p_ctypes], - None, -) -def llama_perf_dump_yaml( - stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / -): - ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 74d73dc85..dc2234408 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 74d73dc85cc2057446bf63cc37ff649ae7cebd80 +Subproject commit dc22344088a7ee81a1e4f096459b03a72f24ccdc From 7ba257e5dac58f8796a12260776f27fd989a1182 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:33:55 -0500 Subject: [PATCH 101/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 32 ++++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index bb2ba1993..24663146c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -221,6 +221,7 @@ # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -249,6 +250,7 @@ LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 +LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 # // note: these values should be synchronized with ggml_rope @@ -392,12 +394,14 @@ # LLAMA_ROPE_SCALING_TYPE_NONE = 0, # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, +# LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, # LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, # }; LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 LLAMA_ROPE_SCALING_TYPE_NONE = 0 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 +LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN # enum llama_pooling_type { @@ -2933,6 +2937,34 @@ def llama_chat_apply_template( ... +# // Get list of built-in chat templates +# LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); +@ctypes_function( + "llama_chat_builtin_templates", + [ + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ], + ctypes.c_int32, +) +def llama_chat_builtin_templates( + output: CtypesArray[bytes], + len: Union[ctypes.c_size_t, int], + /, +) -> int: + """Get list of built-in chat templates. + + Args: + output: Output buffer to store template names. + len: Length of the output buffer. + + Returns: + Number of templates available. + Returns a negative number on error. + """ + ... + + # // # // Sampling API # // diff --git a/vendor/llama.cpp b/vendor/llama.cpp index dc2234408..7736837d6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit dc22344088a7ee81a1e4f096459b03a72f24ccdc +Subproject commit 7736837d62efed1dbebfe579472fca041eda12d6 From 9d06e36c00aea381dd995abc800ce9727da13a82 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:49:20 -0500 Subject: [PATCH 102/155] fix(ci): Explicitly install arm64 python version --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f619cf1f..01b2f4c40 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,6 +126,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + architecture: "arm64" cache: 'pip' - name: Restore model cache @@ -166,6 +167,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.9" + architecture: "arm64" - name: Restore model cache uses: actions/cache@v4 From fb0b8fefd41dc8c467b0e353638f7345254274b5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:53:06 -0500 Subject: [PATCH 103/155] fix(ci): Explicitly set cmake osx architecture --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9f3c2b36..584e54d39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,8 @@ if (LLAMA_BUILD) set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE) set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE) set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE) + + set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build architecture for OS X" FORCE) endif() if (APPLE) From 72ed7b8855a168a454026b98c7f86a452524a7d1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:55:57 -0500 Subject: [PATCH 104/155] fix(ci): Explicitly test on arm64 macos runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 01b2f4c40..445554e6b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-latest + runs-on: macos-14-arm64 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-latest + runs-on: macos-14-arm64 steps: - uses: actions/checkout@v4 with: From 8988aaf7f64a4ffbe30fb1b94d786848ea1efd3c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:02:04 -0500 Subject: [PATCH 105/155] fix(ci): Use macos-14 runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 445554e6b..f5a08f6cb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-14-arm64 + runs-on: macos-14 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14-arm64 + runs-on: macos-14 steps: - uses: actions/checkout@v4 with: From f11a781d86c914ab12dc620edf64c049c6fb0ee7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:04:48 -0500 Subject: [PATCH 106/155] fix(ci): Use macos-13 runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f5a08f6cb..1631d63de 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-14 + runs-on: macos-13 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14 + runs-on: macos-13 steps: - uses: actions/checkout@v4 with: From 9a09fc78487b08423f4e2c9f02b7bc308dc0e2e7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:19:36 -0500 Subject: [PATCH 107/155] fix(ci): Debug print python system architecture --- .github/workflows/test.yaml | 49 ++++++++++++------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1631d63de..ee8680a9b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -51,19 +51,11 @@ jobs: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - name: Test with pytest run: | python -m pytest @@ -90,22 +82,13 @@ jobs: with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: bash - name: Install dependencies (Windows) - if: runner.os == 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd + shell: cmd - name: Test with pytest run: | @@ -129,6 +112,12 @@ jobs: architecture: "arm64" cache: 'pip' + - name: System Info + run: | + uname -a + sysctl -n machdep.cpu.brand_string + python3 -c "import platform; print(platform.machine(), platform.architecture())" + - name: Restore model cache uses: actions/cache@v4 with: @@ -136,21 +125,13 @@ jobs: key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=off" python -m uv pip install .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - - name: Test with pytest run: | python -m pytest @@ -169,25 +150,25 @@ jobs: python-version: "3.9" architecture: "arm64" + - name: System Info + run: | + uname -a + sysctl -n machdep.cpu.brand_string + python3 -c "import platform; print(platform.machine(), platform.architecture())" + - name: Restore model cache uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' + - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install uv CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose - name: Test with pytest run: | python -m pytest From a412ba5539be6a5499acfcb0a2c234665fcbda11 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:38:12 -0500 Subject: [PATCH 108/155] fix(ci): Update config --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee8680a9b..f8b7893f8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -109,7 +109,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: "arm64" cache: 'pip' - name: System Info @@ -138,7 +137,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-13 + runs-on: macos-14 steps: - uses: actions/checkout@v4 with: From df05096fc56347efe14ee550f065a1b15933c322 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:42:33 -0500 Subject: [PATCH 109/155] fix(ci): Install with regular pip --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f8b7893f8..9332e0830 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -164,8 +164,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install uv - CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose shell: bash - name: Test with pytest From 1cd3f2cc6aceaa3167c943eb33b4076ae7a9ba53 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:52:54 -0500 Subject: [PATCH 110/155] fix(ci): gg --- .github/workflows/test.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9332e0830..a4d56110a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -110,6 +110,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + architecture: "arm64" - name: System Info run: | @@ -137,7 +138,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14 + runs-on: macos-13 steps: - uses: actions/checkout@v4 with: @@ -163,10 +164,10 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose + python3 -m pip install --upgrade pip + CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose shell: bash - name: Test with pytest run: | - python -m pytest + python3 -m pytest From b34f20046b9fc058ed8581291a724c6723b06fe4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:59:51 -0500 Subject: [PATCH 111/155] fix(ci): Use python3 --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a4d56110a..7f4774959 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,15 +126,15 @@ jobs: - name: Install dependencies (Linux/MacOS) run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - CMAKE_ARGS="-DLLAMA_METAL=off" python -m uv pip install .[all] --verbose + python3 -m pip install --upgrade pip + python3 -m pip install uv + python3 -m uv pip install -e .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose shell: bash - name: Test with pytest run: | - python -m pytest + python3 -m pytest build-macos-metal: needs: download-model From d8cc231fd00b67528e7e4563c2b4c514acbcd8f9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:06:07 -0500 Subject: [PATCH 112/155] fix(ci): Use default architecture chosen by action --- .github/workflows/test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7f4774959..95f6e5a27 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -110,7 +110,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - architecture: "arm64" - name: System Info run: | @@ -148,7 +147,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.9" - architecture: "arm64" - name: System Info run: | From d5d50990d016333982ff350c9c151efe5732d76b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:14:59 -0500 Subject: [PATCH 113/155] fix(ci): Update CMakeLists.txt for macos --- CMakeLists.txt | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 584e54d39..64a0304a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,20 +62,35 @@ if (LLAMA_BUILD) # Enable building of the common library set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) - # Building llama - if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") - # Need to disable these llama.cpp flags on Apple x86_64, - # otherwise users may encounter invalid instruction errors - set(GGML_AVX "Off" CACHE BOOL "ggml: enable AVX" FORCE) - set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE) - set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE) - set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE) - - set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build architecture for OS X" FORCE) - endif() - + # Architecture detection and settings for Apple platforms if (APPLE) - set(GGML_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) + # Get the target architecture + execute_process( + COMMAND uname -m + OUTPUT_VARIABLE HOST_ARCH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # If CMAKE_OSX_ARCHITECTURES is not set, use the host architecture + if(NOT CMAKE_OSX_ARCHITECTURES) + set(CMAKE_OSX_ARCHITECTURES ${HOST_ARCH} CACHE STRING "Build architecture for macOS" FORCE) + endif() + + message(STATUS "Host architecture: ${HOST_ARCH}") + message(STATUS "Target architecture: ${CMAKE_OSX_ARCHITECTURES}") + + # Configure based on target architecture + if(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64") + # Intel Mac settings + set(GGML_AVX "OFF" CACHE BOOL "ggml: enable AVX" FORCE) + set(GGML_AVX2 "OFF" CACHE BOOL "ggml: enable AVX2" FORCE) + set(GGML_FMA "OFF" CACHE BOOL "ggml: enable FMA" FORCE) + set(GGML_F16C "OFF" CACHE BOOL "ggml: enable F16C" FORCE) + endif() + + # Metal settings (enable for both architectures) + set(GGML_METAL "ON" CACHE BOOL "ggml: enable Metal" FORCE) + set(GGML_METAL_EMBED_LIBRARY "ON" CACHE BOOL "ggml: embed metal library" FORCE) endif() add_subdirectory(vendor/llama.cpp) @@ -130,7 +145,7 @@ if (LLAMA_BUILD) # Building llava add_subdirectory(vendor/llama.cpp/examples/llava) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - # Set CUDA_ARCHITECTURES to OFF on Windows + if (WIN32) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() From 4f17ae5af073f3741be7235f86e2b183db62a260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=CE=9BBL=C3=98=20=E1=84=83=CE=9E?= Date: Fri, 6 Dec 2024 04:31:01 -0500 Subject: [PATCH 114/155] fix(ci): Remove cuda version 12.5.0 incompatibility with VS (#1838) Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 647b171e8..bde626828 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.0") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") 'releasetag' = @("basic") } From 991d9cde1b81dfdda4d3117b508c16c6fe7dbdca Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:32:19 -0500 Subject: [PATCH 115/155] fix(ci): Remove CUDA 12.5 from index --- .github/workflows/generate-index-from-release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 8d80e7e47..51961088d 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,7 +44,6 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From 27953036b1bb1da9e4b71b875f66ba8f1aefbf6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:32:37 -0500 Subject: [PATCH 116/155] chore(deps): bump pypa/cibuildwheel from 2.21.1 to 2.22.0 (#1844) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.1 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.1...v2.22.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index bfa97decd..1d549ed68 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -42,7 +42,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -69,7 +69,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 8e6fb5cb7..01da3cef5 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -43,7 +43,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From 2523472c3eccb9ab9277117cc4ff705212b6888a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:36:22 -0500 Subject: [PATCH 117/155] fix: Fix pickling of Llama class by setting seed from _seed member. Closes #1769 --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d15a88b00..ce1a26649 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2073,7 +2073,7 @@ def __getstate__(self): use_mlock=self.model_params.use_mlock, kv_overrides=self.kv_overrides, # Context Params - seed=self.context_params.seed, + seed=self._seed, n_ctx=self.context_params.n_ctx, n_batch=self.n_batch, n_ubatch=self.context_params.n_ubatch, From ddac04c78e504196207292aeca4c3adccc655a88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:39:27 -0500 Subject: [PATCH 118/155] chore(deps): bump conda-incubator/setup-miniconda from 3.0.4 to 3.1.0 (#1821) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.4 to 3.1.0. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.4...v3.1.0) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index bde626828..3d9c5d9d4 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -59,7 +59,7 @@ jobs: cache: 'pip' - name: Setup Mamba - uses: conda-incubator/setup-miniconda@v3.0.4 + uses: conda-incubator/setup-miniconda@v3.1.0 with: activate-environment: "build" python-version: ${{ matrix.pyver }} From fa04cdc497e2c72888cccf6268836fe3291c3d42 Mon Sep 17 00:00:00 2001 From: ddh0 Date: Fri, 6 Dec 2024 03:40:47 -0600 Subject: [PATCH 119/155] fix logit-bias type hint (#1802) `Optional[Dict[str, float]]` --> `Optional[Dict[int, float]]` --- llama_cpp/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ce1a26649..2fd7ff193 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1141,7 +1141,7 @@ def _create_completion( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[ Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: @@ -1761,7 +1761,7 @@ def create_completion( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -1858,7 +1858,7 @@ def __call__( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -1951,7 +1951,7 @@ def create_chat_completion( model: Optional[str] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, ) -> Union[ From 38fbd2968d3e937c9f4157a54372b02fa527233b Mon Sep 17 00:00:00 2001 From: Rich Dougherty Date: Fri, 6 Dec 2024 22:41:52 +1300 Subject: [PATCH 120/155] docs: Remove ref to llama_eval in llama_cpp.py docs (#1819) The llama_eval function was removed in 0.2.14 (0d37ce5) but the docs still reference it. Updated to match the llama.h docs from upstream. --- llama_cpp/llama_cpp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 24663146c..cd59bb430 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -772,7 +772,7 @@ class llama_context_params(ctypes.Structure): cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache - logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) + logits_all (bool): the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention @@ -2469,10 +2469,10 @@ def llama_synchronize(ctx: llama_context_p, /): "llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) ) def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: - """Token logits obtained from the last call to llama_eval() - The logits for the last token are stored in the last row - Logits for which llama_batch.logits[i] == 0 are undefined - Rows: n_tokens provided with llama_batch + """Token logits obtained from the last call to llama_decode() + The logits for which llama_batch.logits[i] != 0 are stored contiguously + in the order they have appeared in the batch. + Rows: number of tokens for which llama_batch.logits[i] != 0 Cols: n_vocab Returns: From 4192210232ee20ceb028ebd0ee68a6b964f5ffc1 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Fri, 6 Dec 2024 10:47:38 +0100 Subject: [PATCH 121/155] fix: make content not required in ChatCompletionRequestAssistantMessage (#1807) --- llama_cpp/llama_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index bbb58afc3..57836ee76 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -214,7 +214,7 @@ class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): class ChatCompletionRequestAssistantMessage(TypedDict): role: Literal["assistant"] - content: Optional[str] + content: NotRequired[str] tool_calls: NotRequired[ChatCompletionMessageToolCalls] function_call: NotRequired[ ChatCompletionRequestAssistantMessageFunctionCall From 77a12a39663bbe5a5d22a7cc3d5d4f1c591c2851 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 6 Dec 2024 11:01:17 +0100 Subject: [PATCH 122/155] fix: Re-add suport for CUDA 12.5, add CUDA 12.6 (#1775) Add support for Cuda 12.6.1 Update version of Cuda 12.5.0 to 12.5.1 Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3d9c5d9d4..8624ec6d9 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "12.6.1") 'releasetag' = @("basic") } From 073b7e421e419ec389d8c064bbdae26c482ec17c Mon Sep 17 00:00:00 2001 From: "Ignaz \"Ian\" Kraft" Date: Fri, 6 Dec 2024 11:02:34 +0100 Subject: [PATCH 123/155] fix: added missing exit_stack.close() to /v1/chat/completions (#1796) * fix: added missing exit_stack.close() to /v1/chat/completions * fix: added missing exit_stack.close() to /v1/completions --- llama_cpp/server/app.py | 23 ++++++++++++++++------- llama_cpp/server/errors.py | 6 ++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index cd3255176..9b8dce418 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -314,10 +314,14 @@ async def create_completion( else: kwargs["logits_processor"].extend(_min_tokens_logits_processor) - iterator_or_completion: Union[ - llama_cpp.CreateCompletionResponse, - Iterator[llama_cpp.CreateCompletionStreamResponse], - ] = await run_in_threadpool(llama, **kwargs) + try: + iterator_or_completion: Union[ + llama_cpp.CreateCompletionResponse, + Iterator[llama_cpp.CreateCompletionStreamResponse], + ] = await run_in_threadpool(llama, **kwargs) + except Exception as err: + exit_stack.close() + raise err if isinstance(iterator_or_completion, Iterator): # EAFP: It's easier to ask for forgiveness than permission @@ -344,6 +348,7 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: ping_message_factory=_ping_message_factory, ) else: + exit_stack.close() return iterator_or_completion @@ -508,9 +513,13 @@ async def create_chat_completion( else: kwargs["logits_processor"].extend(_min_tokens_logits_processor) - iterator_or_completion: Union[ - llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] - ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) + try: + iterator_or_completion: Union[ + llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] + ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) + except Exception as err: + exit_stack.close() + raise err if isinstance(iterator_or_completion, Iterator): # EAFP: It's easier to ask for forgiveness than permission diff --git a/llama_cpp/server/errors.py b/llama_cpp/server/errors.py index fbf9fd80d..d0eda5664 100644 --- a/llama_cpp/server/errors.py +++ b/llama_cpp/server/errors.py @@ -134,8 +134,6 @@ def error_message_wrapper( ] = None, ) -> Tuple[int, ErrorResponse]: """Wraps error message in OpenAI style error response""" - print(f"Exception: {str(error)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) if body is not None and isinstance( body, ( @@ -149,6 +147,10 @@ def error_message_wrapper( if match is not None: return callback(body, match) + # Only print the trace on unexpected exceptions + print(f"Exception: {str(error)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + # Wrap other errors as internal server error return 500, ErrorResponse( message=str(error), From 9bd0c95424aad6a80a7305761b465006af3dde0d Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Fri, 6 Dec 2024 10:03:41 +0000 Subject: [PATCH 124/155] fix: Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context (#1798) * fix: make use of asyncio to lock llama_proxy context * fix: use aclose instead of close for AsyncExitStack * fix: don't call exit stack close in stream iterator as it will be called by finally from on_complete anyway * fix: use anyio.Lock instead of asyncio.Lock --------- Co-authored-by: Andrei --- llama_cpp/server/app.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 9b8dce418..f7c028475 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -5,7 +5,7 @@ import typing import contextlib -from threading import Lock +from anyio import Lock from functools import partial from typing import Iterator, List, Optional, Union, Dict @@ -70,14 +70,14 @@ def set_llama_proxy(model_settings: List[ModelSettings]): _llama_proxy = LlamaProxy(models=model_settings) -def get_llama_proxy(): +async def get_llama_proxy(): # NOTE: This double lock allows the currently streaming llama model to # check if any other requests are pending in the same thread and cancel # the stream if so. - llama_outer_lock.acquire() + await llama_outer_lock.acquire() release_outer_lock = True try: - llama_inner_lock.acquire() + await llama_inner_lock.acquire() try: llama_outer_lock.release() release_outer_lock = False @@ -159,7 +159,7 @@ async def get_event_publisher( request: Request, inner_send_chan: MemoryObjectSendStream[typing.Any], iterator: Iterator[typing.Any], - on_complete: typing.Optional[typing.Callable[[], None]] = None, + on_complete: typing.Optional[typing.Callable[[], typing.Awaitable[None]]] = None, ): server_settings = next(get_server_settings()) interrupt_requests = ( @@ -182,7 +182,7 @@ async def get_event_publisher( raise e finally: if on_complete: - on_complete() + await on_complete() def _logit_bias_tokens_to_input_ids( @@ -267,10 +267,8 @@ async def create_completion( request: Request, body: CreateCompletionRequest, ) -> llama_cpp.Completion: - exit_stack = contextlib.ExitStack() - llama_proxy = await run_in_threadpool( - lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) - ) + exit_stack = contextlib.AsyncExitStack() + llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -332,7 +330,6 @@ async def create_completion( def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: yield first_response yield from iterator_or_completion - exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -342,13 +339,13 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: request=request, inner_send_chan=send_chan, iterator=iterator(), - on_complete=exit_stack.close, + on_complete=exit_stack.aclose, ), sep="\n", ping_message_factory=_ping_message_factory, ) else: - exit_stack.close() + await exit_stack.aclose() return iterator_or_completion @@ -477,10 +474,8 @@ async def create_chat_completion( # where the dependency is cleaned up before a StreamingResponse # is complete. # https://github.com/tiangolo/fastapi/issues/11143 - exit_stack = contextlib.ExitStack() - llama_proxy = await run_in_threadpool( - lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) - ) + exit_stack = contextlib.AsyncExitStack() + llama_proxy = exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -530,7 +525,6 @@ async def create_chat_completion( def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: yield first_response yield from iterator_or_completion - exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -540,13 +534,13 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: request=request, inner_send_chan=send_chan, iterator=iterator(), - on_complete=exit_stack.close, + on_complete=exit_stack.aclose, ), sep="\n", ping_message_factory=_ping_message_factory, ) else: - exit_stack.close() + await exit_stack.aclose() return iterator_or_completion From 1ea615424b7647a56946ce4fc3ea72b6ebaa867c Mon Sep 17 00:00:00 2001 From: Florents Tselai Date: Fri, 6 Dec 2024 12:11:50 +0200 Subject: [PATCH 125/155] fix(docs): Update development instructions (#1833) --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dbaec5077..e00456580 100644 --- a/README.md +++ b/README.md @@ -752,15 +752,29 @@ pip install --upgrade pip pip install -e . # if you want to use the fastapi / openapi server -pip install -e .[server] +pip install -e '.[server]' # to install all optional dependencies -pip install -e .[all] +pip install -e '.[all]' # to clear the local build cache make clean ``` +Now try running the tests + +```bash +pytest +``` + +There's a `Makefile` available with useful targets. +A typical workflow would look like this: + +```bash +make build +make test +``` + You can also test out specific commits of `llama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require changes to the `llama_cpp/llama_cpp.py` file to match the new API (additional changes may be required elsewhere). From d610477a0659fd7f56c5712adccebde4135a250c Mon Sep 17 00:00:00 2001 From: Luke Stanley <306671+lukestanley@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:22:26 +0000 Subject: [PATCH 126/155] fix(examples): Refactor Batching notebook to use new sampler chain API (#1793) - Replaced deprecated llama_sample_token with llama_sampler_sample - Updated llama_token_to_piece signature to include lstrip and special arguments - Changed llama_load_model_from_file to use positional argument for params (currently does not accept a keyword argument for it) --- examples/notebooks/Batching.ipynb | 557 ++++++++---------------------- 1 file changed, 146 insertions(+), 411 deletions(-) diff --git a/examples/notebooks/Batching.ipynb b/examples/notebooks/Batching.ipynb index 73b28c744..be7fe9b52 100644 --- a/examples/notebooks/Batching.ipynb +++ b/examples/notebooks/Batching.ipynb @@ -6,6 +6,7 @@ "metadata": {}, "outputs": [], "source": [ + "import ctypes\n", "import llama_cpp" ] }, @@ -13,18 +14,7 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ggml_init_cublas: GGML_CUDA_FORCE_MMQ: no\n", - "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n", - "ggml_init_cublas: found 1 CUDA devices:\n", - " Device 0: NVIDIA GeForce RTX 2060, compute capability 7.5\n" - ] - } - ], + "outputs": [], "source": [ "llama_cpp.llama_backend_init(numa=False)" ] @@ -38,354 +28,94 @@ "name": "stderr", "output_type": "stream", "text": [ - "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf (version GGUF V2)\n", - "llama_model_loader: - tensor 0: token_embd.weight q4_K [ 4096, 32000, 1, 1 ]\n", - "llama_model_loader: - tensor 1: output_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 2: output.weight q6_K [ 4096, 32000, 1, 1 ]\n", - "llama_model_loader: - tensor 3: blk.0.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 4: blk.0.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 5: blk.0.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 6: blk.0.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 7: blk.0.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 8: blk.0.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 9: blk.0.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 10: blk.0.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 11: blk.0.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 12: blk.1.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 13: blk.1.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 14: blk.1.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 15: blk.1.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 16: blk.1.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 17: blk.1.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 18: blk.1.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 19: blk.1.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 20: blk.1.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 21: blk.2.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 22: blk.2.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 23: blk.2.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 24: blk.2.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 25: blk.2.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 26: blk.2.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 27: blk.2.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 28: blk.2.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 29: blk.2.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 30: blk.3.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 31: blk.3.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 32: blk.3.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 33: blk.3.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 34: blk.3.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 35: blk.3.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 36: blk.3.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 37: blk.3.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 38: blk.3.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 39: blk.4.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 40: blk.4.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 41: blk.4.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 42: blk.4.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 43: blk.4.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 44: blk.4.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 45: blk.4.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 46: blk.4.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 47: blk.4.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 48: blk.5.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 49: blk.5.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 50: blk.5.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 51: blk.5.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 52: blk.5.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 53: blk.5.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 54: blk.5.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 55: blk.5.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 56: blk.5.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 57: blk.6.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 58: blk.6.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 59: blk.6.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 60: blk.6.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 61: blk.6.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 62: blk.6.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 63: blk.6.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 64: blk.6.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 65: blk.6.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 66: blk.7.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 67: blk.7.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 68: blk.7.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 69: blk.7.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 70: blk.7.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 71: blk.7.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 72: blk.7.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 73: blk.7.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 74: blk.7.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 75: blk.8.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 76: blk.8.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 77: blk.8.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 78: blk.8.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 79: blk.8.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 80: blk.8.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 81: blk.8.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 82: blk.8.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 83: blk.8.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 84: blk.9.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 85: blk.9.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 86: blk.9.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 87: blk.9.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 88: blk.9.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 89: blk.9.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 90: blk.9.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 91: blk.9.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 92: blk.9.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 93: blk.10.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 94: blk.10.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 95: blk.10.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 96: blk.10.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 97: blk.10.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 98: blk.10.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 99: blk.10.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 100: blk.10.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 101: blk.10.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 102: blk.11.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 103: blk.11.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 104: blk.11.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 105: blk.11.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 106: blk.11.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 107: blk.11.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 108: blk.11.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 109: blk.11.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 110: blk.11.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 111: blk.12.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 112: blk.12.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 113: blk.12.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 114: blk.12.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 115: blk.12.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 116: blk.12.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 117: blk.12.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 118: blk.12.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 119: blk.12.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 120: blk.13.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 121: blk.13.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 122: blk.13.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 123: blk.13.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 124: blk.13.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 125: blk.13.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 126: blk.13.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 127: blk.13.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 128: blk.13.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 129: blk.14.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 130: blk.14.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 131: blk.14.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 132: blk.14.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 133: blk.14.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 134: blk.14.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 135: blk.14.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 136: blk.14.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 137: blk.14.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 138: blk.15.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 139: blk.15.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 140: blk.15.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 141: blk.15.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 142: blk.15.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 143: blk.15.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 144: blk.15.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 145: blk.15.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 146: blk.15.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 147: blk.16.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 148: blk.16.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 149: blk.16.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 150: blk.16.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 151: blk.16.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 152: blk.16.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 153: blk.16.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 154: blk.16.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 155: blk.16.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 156: blk.17.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 157: blk.17.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 158: blk.17.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 159: blk.17.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 160: blk.17.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 161: blk.17.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 162: blk.17.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 163: blk.17.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 164: blk.17.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 165: blk.18.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 166: blk.18.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 167: blk.18.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 168: blk.18.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 169: blk.18.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 170: blk.18.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 171: blk.18.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 172: blk.18.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 173: blk.18.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 174: blk.19.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 175: blk.19.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 176: blk.19.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 177: blk.19.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 178: blk.19.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 179: blk.19.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 180: blk.19.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 181: blk.19.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 182: blk.19.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 183: blk.20.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 184: blk.20.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 185: blk.20.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 186: blk.20.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 187: blk.20.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 188: blk.20.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 189: blk.20.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 190: blk.20.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 191: blk.20.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 192: blk.21.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 193: blk.21.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 194: blk.21.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 195: blk.21.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 196: blk.21.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 197: blk.21.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 198: blk.21.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 199: blk.21.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 200: blk.21.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 201: blk.22.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 202: blk.22.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 203: blk.22.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 204: blk.22.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 205: blk.22.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 206: blk.22.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 207: blk.22.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 208: blk.22.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 209: blk.22.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 210: blk.23.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 211: blk.23.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 212: blk.23.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 213: blk.23.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 214: blk.23.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 215: blk.23.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 216: blk.23.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 217: blk.23.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 218: blk.23.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 219: blk.24.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 220: blk.24.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 221: blk.24.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 222: blk.24.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 223: blk.24.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 224: blk.24.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 225: blk.24.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 226: blk.24.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 227: blk.24.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 228: blk.25.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 229: blk.25.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 230: blk.25.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 231: blk.25.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 232: blk.25.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 233: blk.25.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 234: blk.25.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 235: blk.25.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 236: blk.25.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 237: blk.26.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 238: blk.26.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 239: blk.26.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 240: blk.26.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 241: blk.26.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 242: blk.26.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 243: blk.26.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 244: blk.26.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 245: blk.26.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 246: blk.27.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 247: blk.27.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 248: blk.27.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 249: blk.27.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 250: blk.27.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 251: blk.27.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 252: blk.27.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 253: blk.27.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 254: blk.27.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 255: blk.28.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 256: blk.28.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 257: blk.28.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 258: blk.28.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 259: blk.28.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 260: blk.28.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 261: blk.28.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 262: blk.28.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 263: blk.28.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 264: blk.29.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 265: blk.29.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 266: blk.29.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 267: blk.29.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 268: blk.29.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 269: blk.29.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 270: blk.29.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 271: blk.29.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 272: blk.29.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 273: blk.30.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 274: blk.30.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 275: blk.30.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 276: blk.30.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 277: blk.30.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 278: blk.30.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 279: blk.30.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 280: blk.30.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 281: blk.30.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 282: blk.31.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 283: blk.31.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 284: blk.31.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 285: blk.31.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 286: blk.31.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 287: blk.31.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 288: blk.31.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 289: blk.31.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 290: blk.31.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - kv 0: general.architecture str \n", - "llama_model_loader: - kv 1: general.name str \n", - "llama_model_loader: - kv 2: llama.context_length u32 \n", - "llama_model_loader: - kv 3: llama.embedding_length u32 \n", - "llama_model_loader: - kv 4: llama.block_count u32 \n", - "llama_model_loader: - kv 5: llama.feed_forward_length u32 \n", - "llama_model_loader: - kv 6: llama.rope.dimension_count u32 \n", - "llama_model_loader: - kv 7: llama.attention.head_count u32 \n", - "llama_model_loader: - kv 8: llama.attention.head_count_kv u32 \n", - "llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 \n", - "llama_model_loader: - kv 10: general.file_type u32 \n", - "llama_model_loader: - kv 11: tokenizer.ggml.model str \n", - "llama_model_loader: - kv 12: tokenizer.ggml.tokens arr \n", - "llama_model_loader: - kv 13: tokenizer.ggml.scores arr \n", - "llama_model_loader: - kv 14: tokenizer.ggml.token_type arr \n", - "llama_model_loader: - kv 15: general.quantization_version u32 \n", + "llama_model_loader: loaded meta data with 20 key-value pairs and 291 tensors from /workspaces/llama-cpp-python/mistral-7b-v0.1.Q2_K.gguf (version GGUF V2)\n", + "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", + "llama_model_loader: - kv 0: general.architecture str = llama\n", + "llama_model_loader: - kv 1: general.name str = mistralai_mistral-7b-v0.1\n", + "llama_model_loader: - kv 2: llama.context_length u32 = 32768\n", + "llama_model_loader: - kv 3: llama.embedding_length u32 = 4096\n", + "llama_model_loader: - kv 4: llama.block_count u32 = 32\n", + "llama_model_loader: - kv 5: llama.feed_forward_length u32 = 14336\n", + "llama_model_loader: - kv 6: llama.rope.dimension_count u32 = 128\n", + "llama_model_loader: - kv 7: llama.attention.head_count u32 = 32\n", + "llama_model_loader: - kv 8: llama.attention.head_count_kv u32 = 8\n", + "llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000010\n", + "llama_model_loader: - kv 10: llama.rope.freq_base f32 = 10000.000000\n", + "llama_model_loader: - kv 11: general.file_type u32 = 10\n", + "llama_model_loader: - kv 12: tokenizer.ggml.model str = llama\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "llama_model_loader: - kv 13: tokenizer.ggml.tokens arr[str,32000] = [\"\", \"\", \"\", \"<0x00>\", \"<...\n", + "llama_model_loader: - kv 14: tokenizer.ggml.scores arr[f32,32000] = [0.000000, 0.000000, 0.000000, 0.0000...\n", + "llama_model_loader: - kv 15: tokenizer.ggml.token_type arr[i32,32000] = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", + "llama_model_loader: - kv 16: tokenizer.ggml.bos_token_id u32 = 1\n", + "llama_model_loader: - kv 17: tokenizer.ggml.eos_token_id u32 = 2\n", + "llama_model_loader: - kv 18: tokenizer.ggml.unknown_token_id u32 = 0\n", + "llama_model_loader: - kv 19: general.quantization_version u32 = 2\n", "llama_model_loader: - type f32: 65 tensors\n", - "llama_model_loader: - type q4_K: 193 tensors\n", - "llama_model_loader: - type q6_K: 33 tensors\n", - "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n", + "llama_model_loader: - type q2_K: 65 tensors\n", + "llama_model_loader: - type q3_K: 160 tensors\n", + "llama_model_loader: - type q6_K: 1 tensors\n", + "llm_load_vocab: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n", + "llm_load_vocab: special tokens cache size = 3\n", + "llm_load_vocab: token to piece cache size = 0.1637 MB\n", "llm_load_print_meta: format = GGUF V2\n", "llm_load_print_meta: arch = llama\n", "llm_load_print_meta: vocab type = SPM\n", "llm_load_print_meta: n_vocab = 32000\n", "llm_load_print_meta: n_merges = 0\n", - "llm_load_print_meta: n_ctx_train = 4096\n", + "llm_load_print_meta: vocab_only = 0\n", + "llm_load_print_meta: n_ctx_train = 32768\n", "llm_load_print_meta: n_embd = 4096\n", + "llm_load_print_meta: n_layer = 32\n", "llm_load_print_meta: n_head = 32\n", "llm_load_print_meta: n_head_kv = 8\n", - "llm_load_print_meta: n_layer = 32\n", "llm_load_print_meta: n_rot = 128\n", + "llm_load_print_meta: n_swa = 0\n", + "llm_load_print_meta: n_embd_head_k = 128\n", + "llm_load_print_meta: n_embd_head_v = 128\n", "llm_load_print_meta: n_gqa = 4\n", + "llm_load_print_meta: n_embd_k_gqa = 1024\n", + "llm_load_print_meta: n_embd_v_gqa = 1024\n", "llm_load_print_meta: f_norm_eps = 0.0e+00\n", "llm_load_print_meta: f_norm_rms_eps = 1.0e-05\n", "llm_load_print_meta: f_clamp_kqv = 0.0e+00\n", "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n", + "llm_load_print_meta: f_logit_scale = 0.0e+00\n", "llm_load_print_meta: n_ff = 14336\n", + "llm_load_print_meta: n_expert = 0\n", + "llm_load_print_meta: n_expert_used = 0\n", + "llm_load_print_meta: causal attn = 1\n", + "llm_load_print_meta: pooling type = 0\n", + "llm_load_print_meta: rope type = 0\n", + "llm_load_print_meta: rope scaling = linear\n", "llm_load_print_meta: freq_base_train = 10000.0\n", "llm_load_print_meta: freq_scale_train = 1\n", + "llm_load_print_meta: n_ctx_orig_yarn = 32768\n", + "llm_load_print_meta: rope_finetuned = unknown\n", + "llm_load_print_meta: ssm_d_conv = 0\n", + "llm_load_print_meta: ssm_d_inner = 0\n", + "llm_load_print_meta: ssm_d_state = 0\n", + "llm_load_print_meta: ssm_dt_rank = 0\n", + "llm_load_print_meta: ssm_dt_b_c_rms = 0\n", "llm_load_print_meta: model type = 7B\n", - "llm_load_print_meta: model ftype = mostly Q4_K - Medium\n", + "llm_load_print_meta: model ftype = Q2_K - Medium\n", "llm_load_print_meta: model params = 7.24 B\n", - "llm_load_print_meta: model size = 4.07 GiB (4.83 BPW) \n", - "llm_load_print_meta: general.name = LLaMA v2\n", - "llm_load_print_meta: BOS token = 1 ''\n", - "llm_load_print_meta: EOS token = 2 ''\n", - "llm_load_print_meta: UNK token = 0 ''\n", - "llm_load_print_meta: LF token = 13 '<0x0A>'\n", - "llm_load_tensors: ggml ctx size = 0.10 MB\n", - "llm_load_tensors: using CUDA for GPU acceleration\n", - "llm_load_tensors: mem required = 70.41 MB\n", - "llm_load_tensors: offloading 32 repeating layers to GPU\n", - "llm_load_tensors: offloading non-repeating layers to GPU\n", - "llm_load_tensors: offloaded 35/35 layers to GPU\n", - "llm_load_tensors: VRAM used: 4095.05 MB\n", - ".................................................................................................\n" + "llm_load_print_meta: model size = 2.87 GiB (3.41 BPW) \n", + "llm_load_print_meta: general.name = mistralai_mistral-7b-v0.1\n", + "llm_load_print_meta: BOS token = 1 ''\n", + "llm_load_print_meta: EOS token = 2 ''\n", + "llm_load_print_meta: UNK token = 0 ''\n", + "llm_load_print_meta: LF token = 13 '<0x0A>'\n", + "llm_load_print_meta: EOG token = 2 ''\n", + "llm_load_print_meta: max token length = 48\n", + "llm_load_tensors: ggml ctx size = 0.14 MiB\n", + "llm_load_tensors: CPU buffer size = 2939.57 MiB\n", + "..................................................................................................\n" ] } ], @@ -393,7 +123,7 @@ "params = llama_cpp.llama_model_default_params()\n", "params.n_gpu_layers = 35\n", "model = llama_cpp.llama_load_model_from_file(\n", - " b\"../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf\", params=params\n", + " b\"/workspaces/llama-cpp-python/mistral-7b-v0.1.Q2_K.gguf\", params\n", ") # Update this to whatever" ] }, @@ -406,7 +136,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[1, 1014, 2936, 9060, 285, 1142]\n", + "[1, 415, 2936, 9060, 285, 1142]\n", "58\n" ] } @@ -436,17 +166,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "llama_new_context_with_model: n_ctx = 58\n", + "llama_new_context_with_model: n_ctx = 64\n", + "llama_new_context_with_model: n_batch = 32\n", + "llama_new_context_with_model: n_ubatch = 32\n", + "llama_new_context_with_model: flash_attn = 0\n", "llama_new_context_with_model: freq_base = 10000.0\n", "llama_new_context_with_model: freq_scale = 1\n", - "llama_kv_cache_init: offloading v cache to GPU\n", - "llama_kv_cache_init: offloading k cache to GPU\n", - "llama_kv_cache_init: VRAM kv self = 7.25 MB\n", - "llama_new_context_with_model: kv self size = 7.25 MB\n", - "llama_build_graph: non-view tensors processed: 740/740\n", - "llama_new_context_with_model: compute buffer total size = 10.63 MB\n", - "llama_new_context_with_model: VRAM scratch buffer: 4.51 MB\n", - "llama_new_context_with_model: total VRAM used: 4106.81 MB (model: 4095.05 MB, context: 11.76 MB)\n" + "llama_kv_cache_init: CPU KV buffer size = 8.00 MiB\n", + "llama_new_context_with_model: KV self size = 8.00 MiB, K (f16): 4.00 MiB, V (f16): 4.00 MiB\n", + "llama_new_context_with_model: CPU output buffer size = 0.12 MiB\n", + "llama_new_context_with_model: CPU compute buffer size = 5.01 MiB\n", + "llama_new_context_with_model: graph nodes = 1030\n", + "llama_new_context_with_model: graph splits = 1\n" ] } ], @@ -476,7 +207,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ctypes\n", + "\n", "\n", "batch.n_tokens = tokens_len\n", "for i in range(tokens_len):\n", @@ -502,10 +233,35 @@ " llama_cpp.llama_kv_cache_seq_cp(ctx, 0, i, 0, batch.n_tokens)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 9, "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Initialize sampler chain with default parameters\n", + "sparams = llama_cpp.llama_sampler_chain_default_params()\n", + "sampler_chain = llama_cpp.llama_sampler_chain_init(sparams)\n", + "\n", + "# Add top_k, top_p, temperature, and final distribution-based sampler\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_top_k(40))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_top_p(0.9, 1))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_temp(0.4))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_dist(1234)) # Final \"dist\" sampler" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -514,61 +270,59 @@ "7\n", "[' j', ' jumped']\n", "8\n", - "[' jumps', ' jumped over']\n", + "[' j over', ' jumped over']\n", "9\n", - "[' jumps over', ' jumped over the']\n", + "[' j over the', ' jumped over the']\n", "10\n", - "[' jumps over the', ' jumped over the lazy']\n", + "[' j over the lazy', ' jumped over the lazy']\n", "11\n", - "[' jumps over the lazy', ' jumped over the lazy dog']\n", + "[' j over the lazy dog', ' jumped over the lazy dog']\n", "12\n", - "[' jumps over the lazy dog', ' jumped over the lazy dog.']\n", + "[' j over the lazy dog.', ' jumped over the lazy dog\\n']\n", "13\n", - "[' jumps over the lazy dog.', ' jumped over the lazy dog.\\n']\n", + "[' j over the lazy dog. También', ' jumped over the lazy dog\\nGroupLayout']\n", "14\n", - "[' jumps over the lazy dog.\\n', ' jumped over the lazy dog.\\n\\n']\n", + "[' j over the lazy dog. También:', ' jumped over the lazy dog\\nGroupLayouting']\n", "15\n", - "[' jumps over the lazy dog.\\n\\n', ' jumped over the lazy dog.\\n\\nThe']\n", + "[' j over the lazy dog. También: is', ' jumped over the lazy dog\\nGroupLayouting is']\n", "16\n", - "[' jumps over the lazy dog.\\n\\nI', ' jumped over the lazy dog.\\n\\nThe quick']\n", + "[' j over the lazy dog. También: is a', ' jumped over the lazy dog\\nGroupLayouting is a']\n", "17\n", - "[' jumps over the lazy dog.\\n\\nI’', ' jumped over the lazy dog.\\n\\nThe quick brown']\n", + "[' j over the lazy dog. También: is a technique', ' jumped over the lazy dog\\nGroupLayouting is a common']\n", "18\n", - "[' jumps over the lazy dog.\\n\\nI’m', ' jumped over the lazy dog.\\n\\nThe quick brown f']\n", + "[' j over the lazy dog. También: is a technique practice', ' jumped over the lazy dog\\nGroupLayouting is a common practice']\n", "19\n", - "[' jumps over the lazy dog.\\n\\nI’m not', ' jumped over the lazy dog.\\n\\nThe quick brown fox']\n", + "[' j over the lazy dog. También: is a technique practice in', ' jumped over the lazy dog\\nGroupLayouting is a common practice in']\n", "20\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped']\n", + "[' j over the lazy dog. También: is a technique practice in the', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the']\n", "21\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over']\n", + "[' j over the lazy dog. También: is a technique practice in the real', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media']\n", "22\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the']\n", + "[' j over the lazy dog. También: is a technique practice in the real-', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry']\n", "23\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy']\n", + "[' j over the lazy dog. También: is a technique practice in the real-.', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry.']\n", "24\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However']\n", "25\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We,', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However,']\n", "26\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there']\n", "27\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\n']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has']\n", "28\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been']\n", "29\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little']\n", "30\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little emp']\n", "31\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown f']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical']\n", "32\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English language', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown fox']\n" + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical research']\n" ] } ], "source": [ - "import ctypes\n", - "\n", "streams = [\"\"] * n_parallel\n", "i_batch = [batch.n_tokens - 1] * n_parallel\n", "\n", @@ -581,36 +335,17 @@ " if i_batch[i] < 0:\n", " continue\n", "\n", - " n_vocab = llama_cpp.llama_n_vocab(model)\n", - " logits = llama_cpp.llama_get_logits_ith(ctx, i_batch[i])\n", - "\n", - " candidates = (llama_cpp.llama_token_data * n_vocab)()\n", - "\n", - " for token_id in range(n_vocab):\n", - " candidates[token_id].id = token_id\n", - " candidates[token_id].logit = logits[token_id]\n", - " candidates[token_id].p = 0.0\n", - "\n", - " candidates_p = llama_cpp.llama_token_data_array(\n", - " candidates, len(candidates), False\n", - " )\n", - "\n", - " top_k = 40\n", - " top_p = 0.9\n", - " temp = 0.4\n", - "\n", - " llama_cpp.llama_sample_top_k(ctx, ctypes.byref(candidates_p), top_k, 1)\n", - " llama_cpp.llama_sample_top_p(ctx, ctypes.byref(candidates_p), top_p, 1)\n", - " llama_cpp.llama_sample_temp(ctx, ctypes.byref(candidates_p), temp)\n", - "\n", - " new_token_id = llama_cpp.llama_sample_token(ctx, ctypes.byref(candidates_p))\n", + " # Sample the next token using the sampler chain\n", + " new_token_id = llama_cpp.llama_sampler_sample(sampler_chain, ctx, -1)\n", "\n", " if new_token_id == llama_cpp.llama_token_eos(ctx) or n_cur == n_len:\n", " i_batch[i] = -1\n", " continue\n", "\n", " buf = (ctypes.c_char * 32)()\n", - " outlen = llama_cpp.llama_token_to_piece(model, new_token_id, buf, len(buf))\n", + " \n", + " # Convert token ID to text\n", + " outlen = llama_cpp.llama_token_to_piece(model, new_token_id, buf, len(buf), 0, False)\n", " streams[i] += bytes(buf[:outlen]).decode(\"utf-8\")\n", "\n", " batch.token[batch.n_tokens] = new_token_id\n", @@ -637,14 +372,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English language', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown fox']\n" + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical research']\n" ] } ], @@ -654,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -663,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -672,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -681,7 +416,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -705,7 +440,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -719,7 +454,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5+" + "version": "3.12.1" }, "orig_nbformat": 4 }, From 4f0ec659f9519d3009873f6adf6aa8af5ab32d8b Mon Sep 17 00:00:00 2001 From: Adam Jones Date: Fri, 6 Dec 2024 12:35:46 +0000 Subject: [PATCH 127/155] fix: chat API logprobs format (#1788) * fix: chat API logprobs format * Fix optional properties --- Makefile | 2 +- llama_cpp/llama_chat_format.py | 53 ++++++++++++++++++++++++---------- llama_cpp/llama_types.py | 22 ++++++++++++-- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index be9b55a3e..52b17b61a 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . run-server: - uvicorn --factory llama.server:app --host ${HOST} --port ${PORT} + python llama_cpp/server --model ${MODEL} clean: - cd vendor/llama.cpp && make clean diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index dfb0af65e..17575c700 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -259,6 +259,31 @@ def to_chat_handler(self) -> LlamaChatCompletionHandler: return chat_formatter_to_chat_completion_handler(self) +def _convert_text_completion_logprobs_to_chat( + logprobs: Optional[llama_types.CompletionLogprobs], +) -> llama_types.ChatCompletionLogprobs: + if logprobs is None: + return None + + return { + "content": [ + { + "token": token, + "bytes": None, + "logprob": logprob, + "top_logprobs": [ + { + "token": top_token, + "logprob": top_logprob, + "bytes": None, + } + for top_token, top_logprob in top_logprobs.items() + ], + } for (token, logprob, top_logprobs) in zip(logprobs["tokens"], logprobs["token_logprobs"], logprobs["top_logprobs"]) + ], + "refusal": None, + } + def _convert_text_completion_to_chat( completion: llama_types.Completion, ) -> llama_types.ChatCompletion: @@ -275,7 +300,7 @@ def _convert_text_completion_to_chat( "role": "assistant", "content": completion["choices"][0]["text"], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": completion["choices"][0]["finish_reason"], } ], @@ -319,7 +344,7 @@ def _convert_text_completion_chunks_to_chat( if chunk["choices"][0]["finish_reason"] is None else {} ), - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "finish_reason": chunk["choices"][0]["finish_reason"], } ], @@ -382,7 +407,7 @@ def _convert_completion_to_chat_function( } ], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": "tool_calls", } ], @@ -435,7 +460,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -472,7 +497,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -1716,7 +1741,7 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": "tool_calls", } ], @@ -2128,7 +2153,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -2230,7 +2255,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": None, @@ -2268,9 +2293,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0][ - "logprobs" - ], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": buffer.pop(0), @@ -2293,7 +2316,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": ( @@ -2379,7 +2402,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -2613,7 +2636,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "message": { "role": "assistant", "content": None if content == "" else content, @@ -3745,7 +3768,7 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "message": { "role": "assistant", "content": None, diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 57836ee76..f647822ff 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -82,10 +82,28 @@ class ChatCompletionFunction(TypedDict): parameters: Dict[str, JsonType] # TODO: make this more specific +class ChatCompletionTopLogprobToken(TypedDict): + token: str + logprob: float + bytes: Optional[List[int]] + + +class ChatCompletionLogprobToken(ChatCompletionTopLogprobToken): + token: str + logprob: float + bytes: Optional[List[int]] + top_logprobs: List[ChatCompletionTopLogprobToken] + + +class ChatCompletionLogprobs(TypedDict): + content: Optional[List[ChatCompletionLogprobToken]] + refusal: Optional[List[ChatCompletionLogprobToken]] + + class ChatCompletionResponseChoice(TypedDict): index: int message: "ChatCompletionResponseMessage" - logprobs: Optional[CompletionLogprobs] + logprobs: Optional[ChatCompletionLogprobs] finish_reason: Optional[str] @@ -134,7 +152,7 @@ class ChatCompletionStreamResponseChoice(TypedDict): ChatCompletionStreamResponseDelta, ChatCompletionStreamResponseDeltaEmpty ] finish_reason: Optional[Literal["stop", "length", "tool_calls", "function_call"]] - logprobs: NotRequired[Optional[CompletionLogprobs]] + logprobs: NotRequired[Optional[ChatCompletionLogprobs]] class CreateChatCompletionStreamResponse(TypedDict): From df136cb3c7d3a7a9a4e30ef2d16c5e42e0e36e43 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 07:36:47 -0500 Subject: [PATCH 128/155] misc: Update development Makefile --- Makefile | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index be9b55a3e..37ef9b1d6 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,15 @@ build.debug: --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-ggdb -O0';-DCMAKE_CXX_FLAGS='-ggdb -O0'" \ --editable . +build.debug.extra: + python3 -m pip install \ + --verbose \ + --config-settings=cmake.verbose=true \ + --config-settings=logging.level=INFO \ + --config-settings=install.strip=false \ + --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-fsanitize=address -ggdb -O0';-DCMAKE_CXX_FLAGS='-fsanitize=address -ggdb -O0'" \ + --editable . + build.cuda: CMAKE_ARGS="-DGGML_CUDA=on" python3 -m pip install --verbose -e . @@ -46,7 +55,7 @@ build.rpc: CMAKE_ARGS="-DGGML_RPC=on" python3 -m pip install --verbose -e . build.sdist: - python3 -m build --sdist + python3 -m build --sdist --verbose deploy.pypi: python3 -m twine upload dist/* @@ -56,7 +65,7 @@ deploy.gh-docs: mkdocs gh-deploy test: - python3 -m pytest + python3 -m pytest --full-trace -v docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . @@ -68,11 +77,11 @@ clean: - cd vendor/llama.cpp && make clean - cd vendor/llama.cpp && rm libllama.so - rm -rf _skbuild - - rm llama_cpp/*.so - - rm llama_cpp/*.dylib - - rm llama_cpp/*.metal - - rm llama_cpp/*.dll - - rm llama_cpp/*.lib + - rm llama_cpp/lib/*.so + - rm llama_cpp/lib/*.dylib + - rm llama_cpp/lib/*.metal + - rm llama_cpp/lib/*.dll + - rm llama_cpp/lib/*.lib .PHONY: \ update \ From b9b50e50b9d29b2034c50e1b0ee09dc7c47bbc63 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 07:37:53 -0500 Subject: [PATCH 129/155] misc: Update run server command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 78b11cf11..26ddf2c7a 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . run-server: - python llama_cpp/server --model ${MODEL} + python3 -m llama_cpp.server --model ${MODEL} clean: - cd vendor/llama.cpp && make clean From 5585f8afe1aea74dca7364f18a76f0426381001d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:40:19 -0500 Subject: [PATCH 130/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 12 ++++++------ vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cd59bb430..44a1461db 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -344,9 +344,9 @@ # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors # LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack +# //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack +# //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack # LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # @@ -382,9 +382,9 @@ LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 LLAMA_FTYPE_MOSTLY_IQ1_M = 31 LLAMA_FTYPE_MOSTLY_BF16 = 32 -LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 -LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 -LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 +# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 +# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 +# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_GUESSED = 1024 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7736837d6..ce8784bdb 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7736837d62efed1dbebfe579472fca041eda12d6 +Subproject commit ce8784bdb153ff7794dde5a50b0ebfa51baa6171 From 61508c265a4e295d69aaee7533bde5b51278e574 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:55:03 -0500 Subject: [PATCH 131/155] Add CUDA 12.5 and 12.6 to generated output wheels --- .github/workflows/generate-index-from-release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 51961088d..5c7171bc1 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,6 +44,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From a9fe0f872527ffc1bf64e125b7996ec4dfa598eb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:56:45 -0500 Subject: [PATCH 132/155] chore: Bump version --- CHANGELOG.md | 14 ++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a194fece..331161a0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.3] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ce8784bdb153ff7794dde5a50b0ebfa51baa6171 +- fix: chat API logprobs format by @domdomegg in #1788 +- feat: Add support for CUDA 12.6, fix CUDA 12.5 by @Smartappli in #1775 +- fix: Make content not required in ChatCompletionRequestAssistantMessage by @feloy in #1807 +- fix: Fix pickling of Llama class by setting seed from _seed member by @abetlen in 2523472c3eccb9ab9277117cc4ff705212b6888a +- fix: Fix logit-bias type hint by @ddh0 in #1802 +- fix(server): Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context by @gjpower in #1798 +- fix(server): Added missing exit_stack.close() to /v1/chat/completions by @Ian321 in #1796 +- fix(examples): Refactor Batching notebook to use new sampler chain API by @lukestanley in #1793 +- fix(docs): Update development instructions by @Florents-Tselai in #1833 +- fix(docs): Remove ref to llama_eval in llama_cpp.py docs by @richdougherty in #1819 + ## [0.3.2] - feat: Update llama.cpp to ggerganov/llama.cpp@74d73dc85cc2057446bf63cc37ff649ae7cebd80 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index cda38a841..36c0eafc0 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.2" +__version__ = "0.3.3" From ca808028bd16b8327bd84128d48015a4b1304690 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 23:51:49 -0500 Subject: [PATCH 133/155] fix(ci): hotfix for wheels --- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/generate-index-from-release.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 8624ec6d9..3d410148f 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "12.6.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1") 'releasetag' = @("basic") } diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 01da3cef5..9b97bf2f5 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-12, macos-13, macos-14] + os: [macos-13, macos-14, macos-15] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 5c7171bc1..255ee67d6 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,8 +44,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From 002f58300fdaf865cf1d305b3721c5d96003c440 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 23:53:30 -0500 Subject: [PATCH 134/155] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331161a0c..2a1d3225f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.4] + +- fix(ci): Build wheels for macos 13-15, cuda 12.1-12.4 by @abetlen in ca808028bd16b8327bd84128d48015a4b1304690 + ## [0.3.3] - feat: Update llama.cpp to ggerganov/llama.cpp@ce8784bdb153ff7794dde5a50b0ebfa51baa6171 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 36c0eafc0..dd8f1fc58 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.3" +__version__ = "0.3.4" From ea4d86ac25e0f39370ff3f54550387a02467ab77 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 00:42:15 -0500 Subject: [PATCH 135/155] fix(ci): update macos runner image to non-deprecated version --- .github/workflows/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 1d549ed68..7307c85ab 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-12] + os: [ubuntu-20.04, windows-2019, macos-13] steps: - uses: actions/checkout@v4 From afedfc888462f9a6e809dc9455eb3b663764cc3f Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Mon, 9 Dec 2024 23:47:18 +0000 Subject: [PATCH 136/155] fix: add missing await statements for async exit_stack handling (#1858) --- llama_cpp/server/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index f7c028475..b6db453b8 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -318,7 +318,7 @@ async def create_completion( Iterator[llama_cpp.CreateCompletionStreamResponse], ] = await run_in_threadpool(llama, **kwargs) except Exception as err: - exit_stack.close() + await exit_stack.aclose() raise err if isinstance(iterator_or_completion, Iterator): @@ -475,7 +475,7 @@ async def create_chat_completion( # is complete. # https://github.com/tiangolo/fastapi/issues/11143 exit_stack = contextlib.AsyncExitStack() - llama_proxy = exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) + llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -513,7 +513,7 @@ async def create_chat_completion( llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) except Exception as err: - exit_stack.close() + await exit_stack.aclose() raise err if isinstance(iterator_or_completion, Iterator): From 801a73a621c6ba8f9a8ffe110766c9e721b09520 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 18:49:27 -0500 Subject: [PATCH 137/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ce8784bdb..26a8406ba 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ce8784bdb153ff7794dde5a50b0ebfa51baa6171 +Subproject commit 26a8406ba9198eb6fdd8329fa717555b4f77f05f From 803924b02cc8ca7531b70e6497d1ff7b83f75737 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 18:59:22 -0500 Subject: [PATCH 138/155] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1d3225f..6f6355893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.5] + +- feat: Update llama.cpp to ggerganov/llama.cpp@26a8406ba9198eb6fdd8329fa717555b4f77f05f +- fix(ci): Fix release by updating macos runner image to non-deprecated version by @abetlen in afedfc888462f9a6e809dc9455eb3b663764cc3f +- fix(server): add missing await statements for async exit_stack handling by @gjpower in #1858 + ## [0.3.4] - fix(ci): Build wheels for macos 13-15, cuda 12.1-12.4 by @abetlen in ca808028bd16b8327bd84128d48015a4b1304690 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index dd8f1fc58..d7d9be881 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.4" +__version__ = "0.3.5" From 2bc1d97c9672320828e70dc8293d5f8754682109 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 19 Dec 2024 01:55:12 -0500 Subject: [PATCH 139/155] feat: Update llama.cpp --- llama_cpp/_internals.py | 5 ---- llama_cpp/llama_cpp.py | 54 +++++++++++------------------------------ vendor/llama.cpp | 2 +- 3 files changed, 15 insertions(+), 46 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 994d5f149..8fa2b447f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -805,15 +805,10 @@ def add_penalties( ignore_eos: bool, ): sampler = llama_cpp.llama_sampler_init_penalties( - n_vocab, - special_eos_id, - linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, - penalize_nl, - ignore_eos, ) self._add_sampler(sampler) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 44a1461db..0481cdbcd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -256,13 +256,17 @@ # // note: these values should be synchronized with ggml_rope # // TODO: maybe move this enum to ggml.h (ggml_rope_type) # enum llama_rope_type { -# LLAMA_ROPE_TYPE_NONE = -1, -# LLAMA_ROPE_TYPE_NORM = 0, -# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, +# LLAMA_ROPE_TYPE_NONE = -1, +# LLAMA_ROPE_TYPE_NORM = 0, +# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, +# LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, +# LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, # }; LLAMA_ROPE_TYPE_NONE = -1 LLAMA_ROPE_TYPE_NORM = 0 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 +LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8 +LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file @@ -1265,6 +1269,7 @@ def llama_rope_freq_scale_train(model: llama_model_p, /) -> float: # // Functions to access the model's GGUF metadata scalar values # // - The functions return the length of the string on success, or -1 on failure # // - The output string is always null-terminated and cleared on failure +# // - When retrieving a string, an extra byte must be allocated to account for the null terminator # // - GGUF array values are not supported by these functions @@ -1378,18 +1383,6 @@ def llama_model_n_params(model: llama_model_p, /) -> int: ... -# // Get a llama model tensor -# LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name); -@ctypes_function( - "llama_get_model_tensor", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_void_p -) -def llama_get_model_tensor( - model: llama_model_p, name: Union[ctypes.c_char_p, bytes], / -) -> ctypes.c_void_p: - """Get a llama model tensor""" - ... - - # // Returns true if the model contains an encoder that requires llama_encode() call # LLAMA_API bool llama_model_has_encoder(const struct llama_model * model); @ctypes_function("llama_model_has_encoder", [llama_model_p_ctypes], ctypes.c_bool) @@ -3336,41 +3329,22 @@ def llama_sampler_init_grammar( ... +# /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( -# int32_t n_vocab, // llama_n_vocab() -# llama_token special_eos_id, // llama_token_eos() -# llama_token linefeed_id, // llama_token_nl() -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present, // 0.0 = disabled -# bool penalize_nl, // consider newlines as a repeatable token -# bool ignore_eos); // ignore the end-of-sequence token +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# float penalty_repeat, // 1.0 = disabled +# float penalty_freq, // 0.0 = disabled +# float penalty_present); // 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ - ctypes.c_int32, - llama_token, - llama_token, - ctypes.c_int32, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ctypes.c_bool, - ctypes.c_bool, - ], + [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( - n_vocab: int, - special_eos_id: int, - linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, - penalize_nl: bool, - ignore_eos: bool, /, ) -> llama_sampler_p: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 26a8406ba..7909e8588 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 26a8406ba9198eb6fdd8329fa717555b4f77f05f +Subproject commit 7909e8588ddf70820adf1f325490eb3f67b32875 From c9dfad4b6fdebc85f8f7914e8e46c6d9299ff4b4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 30 Dec 2024 13:43:46 -0500 Subject: [PATCH 140/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7909e8588..716bd6dec 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7909e8588ddf70820adf1f325490eb3f67b32875 +Subproject commit 716bd6dec3e044e5c325386b5b0483392b24cefe From 1d5f5340e49d90bc097c1f75605a4dd7408c2378 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 Jan 2025 16:33:53 -0500 Subject: [PATCH 141/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 31 +++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0481cdbcd..16c6b9d73 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -222,6 +222,7 @@ # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -251,6 +252,7 @@ LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 +LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 # // note: these values should be synchronized with ggml_rope @@ -1090,9 +1092,10 @@ def llama_backend_free(): ... -# LLAMA_API struct llama_model * llama_load_model_from_file( +# DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( # const char * path_model, -# struct llama_model_params params); +# struct llama_model_params params), +# "use llama_model_load_from_file instead"); @ctypes_function( "llama_load_model_from_file", [ctypes.c_char_p, llama_model_params], @@ -1104,6 +1107,20 @@ def llama_load_model_from_file( ... +# LLAMA_API struct llama_model * llama_model_load_from_file( +# const char * path_model, +# struct llama_model_params params); +@ctypes_function( + "llama_model_load_from_file", + [ctypes.c_char_p, llama_model_params], + llama_model_p_ctypes, +) +def llama_model_load_from_file( + path_model: bytes, params: llama_model_params, / +) -> Optional[llama_model_p]: + ... + + # LLAMA_API void llama_free_model(struct llama_model * model); @ctypes_function( "llama_free_model", @@ -1114,6 +1131,16 @@ def llama_free_model(model: llama_model_p, /): ... +# LLAMA_API void llama_model_free(struct llama_model * model); +@ctypes_function( + "llama_model_free", + [llama_model_p_ctypes], + None, +) +def llama_model_free(model: llama_model_p, /): + ... + + # LLAMA_API struct llama_context * llama_new_context_with_model( # struct llama_model * model, # struct llama_context_params params); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 716bd6dec..f7cd13301 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 716bd6dec3e044e5c325386b5b0483392b24cefe +Subproject commit f7cd13301c2a88f97073fd119072b4cc92c08df1 From e8f14ceff9b3279c67368d85f9457bb27168d126 Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Wed, 8 Jan 2025 21:46:43 +0000 Subject: [PATCH 142/155] fix: streaming resource lock (#1879) * fix: correct issue with handling lock during streaming move locking for streaming into get_event_publisher call so it is locked and unlocked in the correct task for the streaming reponse * fix: simplify exit stack management for create_chat_completion and create_completion * fix: correct missing `async with` and format code * fix: remove unnecessary explicit use of AsyncExitStack fix: correct type hints for body_model --------- Co-authored-by: Andrei --- llama_cpp/server/app.py | 224 ++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 121 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index b6db453b8..5120f2416 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -7,7 +7,7 @@ from anyio import Lock from functools import partial -from typing import Iterator, List, Optional, Union, Dict +from typing import List, Optional, Union, Dict import llama_cpp @@ -155,34 +155,71 @@ def create_app( return app +def prepare_request_resources( + body: CreateCompletionRequest | CreateChatCompletionRequest, + llama_proxy: LlamaProxy, + body_model: str | None, + kwargs, +) -> llama_cpp.Llama: + if llama_proxy is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service is not available", + ) + llama = llama_proxy(body_model) + if body.logit_bias is not None: + kwargs["logit_bias"] = ( + _logit_bias_tokens_to_input_ids(llama, body.logit_bias) + if body.logit_bias_type == "tokens" + else body.logit_bias + ) + + if body.grammar is not None: + kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) + + if body.min_tokens > 0: + _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( + [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] + ) + if "logits_processor" not in kwargs: + kwargs["logits_processor"] = _min_tokens_logits_processor + else: + kwargs["logits_processor"].extend(_min_tokens_logits_processor) + return llama + + async def get_event_publisher( request: Request, inner_send_chan: MemoryObjectSendStream[typing.Any], - iterator: Iterator[typing.Any], - on_complete: typing.Optional[typing.Callable[[], typing.Awaitable[None]]] = None, + body: CreateCompletionRequest | CreateChatCompletionRequest, + body_model: str | None, + llama_call, + kwargs, ): server_settings = next(get_server_settings()) interrupt_requests = ( server_settings.interrupt_requests if server_settings else False ) - async with inner_send_chan: - try: - async for chunk in iterate_in_threadpool(iterator): - await inner_send_chan.send(dict(data=json.dumps(chunk))) - if await request.is_disconnected(): - raise anyio.get_cancelled_exc_class()() - if interrupt_requests and llama_outer_lock.locked(): - await inner_send_chan.send(dict(data="[DONE]")) - raise anyio.get_cancelled_exc_class()() - await inner_send_chan.send(dict(data="[DONE]")) - except anyio.get_cancelled_exc_class() as e: - print("disconnected") - with anyio.move_on_after(1, shield=True): - print(f"Disconnected from client (via refresh/close) {request.client}") - raise e - finally: - if on_complete: - await on_complete() + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + async with inner_send_chan: + try: + iterator = await run_in_threadpool(llama_call, llama, **kwargs) + async for chunk in iterate_in_threadpool(iterator): + await inner_send_chan.send(dict(data=json.dumps(chunk))) + if await request.is_disconnected(): + raise anyio.get_cancelled_exc_class()() + if interrupt_requests and llama_outer_lock.locked(): + await inner_send_chan.send(dict(data="[DONE]")) + raise anyio.get_cancelled_exc_class()() + await inner_send_chan.send(dict(data="[DONE]")) + except anyio.get_cancelled_exc_class() as e: + print("disconnected") + with anyio.move_on_after(1, shield=True): + print( + f"Disconnected from client (via refresh/close) {request.client}" + ) + raise e def _logit_bias_tokens_to_input_ids( @@ -267,18 +304,11 @@ async def create_completion( request: Request, body: CreateCompletionRequest, ) -> llama_cpp.Completion: - exit_stack = contextlib.AsyncExitStack() - llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) - if llama_proxy is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Service is not available", - ) if isinstance(body.prompt, list): assert len(body.prompt) <= 1 body.prompt = body.prompt[0] if len(body.prompt) > 0 else "" - llama = llama_proxy( + body_model = ( body.model if request.url.path != "/v1/engines/copilot-codex/completions" else "copilot-codex" @@ -293,44 +323,8 @@ async def create_completion( } kwargs = body.model_dump(exclude=exclude) - if body.logit_bias is not None: - kwargs["logit_bias"] = ( - _logit_bias_tokens_to_input_ids(llama, body.logit_bias) - if body.logit_bias_type == "tokens" - else body.logit_bias - ) - - if body.grammar is not None: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - - if body.min_tokens > 0: - _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( - [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] - ) - if "logits_processor" not in kwargs: - kwargs["logits_processor"] = _min_tokens_logits_processor - else: - kwargs["logits_processor"].extend(_min_tokens_logits_processor) - - try: - iterator_or_completion: Union[ - llama_cpp.CreateCompletionResponse, - Iterator[llama_cpp.CreateCompletionStreamResponse], - ] = await run_in_threadpool(llama, **kwargs) - except Exception as err: - await exit_stack.aclose() - raise err - - if isinstance(iterator_or_completion, Iterator): - # EAFP: It's easier to ask for forgiveness than permission - first_response = await run_in_threadpool(next, iterator_or_completion) - - # If no exception was raised from first_response, we can assume that - # the iterator is valid and we can use it to stream the response. - def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: - yield first_response - yield from iterator_or_completion - + # handle streaming request + if kwargs.get("stream", False): send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( recv_chan, @@ -338,15 +332,29 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: get_event_publisher, request=request, inner_send_chan=send_chan, - iterator=iterator(), - on_complete=exit_stack.aclose, + body=body, + body_model=body_model, + llama_call=llama_cpp.Llama.__call__, + kwargs=kwargs, ), sep="\n", ping_message_factory=_ping_message_factory, ) - else: - await exit_stack.aclose() - return iterator_or_completion + + # handle regular request + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + + if await request.is_disconnected(): + print( + f"Disconnected from client (via refresh/close) before llm invoked {request.client}" + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client closed request", + ) + + return await run_in_threadpool(llama, **kwargs) @router.post( @@ -474,13 +482,8 @@ async def create_chat_completion( # where the dependency is cleaned up before a StreamingResponse # is complete. # https://github.com/tiangolo/fastapi/issues/11143 - exit_stack = contextlib.AsyncExitStack() - llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) - if llama_proxy is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Service is not available", - ) + + body_model = body.model exclude = { "n", "logit_bias_type", @@ -488,44 +491,9 @@ async def create_chat_completion( "min_tokens", } kwargs = body.model_dump(exclude=exclude) - llama = llama_proxy(body.model) - if body.logit_bias is not None: - kwargs["logit_bias"] = ( - _logit_bias_tokens_to_input_ids(llama, body.logit_bias) - if body.logit_bias_type == "tokens" - else body.logit_bias - ) - - if body.grammar is not None: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - - if body.min_tokens > 0: - _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( - [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] - ) - if "logits_processor" not in kwargs: - kwargs["logits_processor"] = _min_tokens_logits_processor - else: - kwargs["logits_processor"].extend(_min_tokens_logits_processor) - - try: - iterator_or_completion: Union[ - llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] - ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) - except Exception as err: - await exit_stack.aclose() - raise err - - if isinstance(iterator_or_completion, Iterator): - # EAFP: It's easier to ask for forgiveness than permission - first_response = await run_in_threadpool(next, iterator_or_completion) - - # If no exception was raised from first_response, we can assume that - # the iterator is valid and we can use it to stream the response. - def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: - yield first_response - yield from iterator_or_completion + # handle streaming request + if kwargs.get("stream", False): send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( recv_chan, @@ -533,15 +501,29 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: get_event_publisher, request=request, inner_send_chan=send_chan, - iterator=iterator(), - on_complete=exit_stack.aclose, + body=body, + body_model=body_model, + llama_call=llama_cpp.Llama.create_chat_completion, + kwargs=kwargs, ), sep="\n", ping_message_factory=_ping_message_factory, ) - else: - await exit_stack.aclose() - return iterator_or_completion + + # handle regular request + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + + if await request.is_disconnected(): + print( + f"Disconnected from client (via refresh/close) before llm invoked {request.client}" + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client closed request", + ) + + return await run_in_threadpool(llama.create_chat_completion, **kwargs) @router.get( From 0580cf273debf4a7f2efcdfd5ef092ff5cedf9b0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 Jan 2025 16:53:54 -0500 Subject: [PATCH 143/155] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f6355893..b4f49c904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.6] + +- feat: Update llama.cpp to ggerganov/llama.cpp@f7cd13301c2a88f97073fd119072b4cc92c08df1 +- fix(server): streaming resource lock by @gjpower in #1879 + ## [0.3.5] - feat: Update llama.cpp to ggerganov/llama.cpp@26a8406ba9198eb6fdd8329fa717555b4f77f05f diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d7d9be881..8c6118fb4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.5" +__version__ = "0.3.6" From 80be68ac36b458e02f5e6af8bddd005a2f680e39 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 21:38:05 -0500 Subject: [PATCH 144/155] feat: Update llama.cpp --- llama_cpp/_internals.py | 181 +++++----- llama_cpp/llama.py | 16 +- llama_cpp/llama_cpp.py | 707 +++++++++++++++++++++++++++++----------- vendor/llama.cpp | 2 +- 4 files changed, 626 insertions(+), 280 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 8fa2b447f..343581dce 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -55,7 +55,13 @@ def __init__( if model is None: raise ValueError(f"Failed to load model from file: {path_model}") + vocab = llama_cpp.llama_model_get_vocab(model) + + if vocab is None: + raise ValueError(f"Failed to get vocab from model: {path_model}") + self.model = model + self.vocab = vocab def free_model(): if self.model is None: @@ -75,7 +81,7 @@ def vocab_type(self) -> int: return llama_cpp.llama_vocab_type(self.model) def n_vocab(self) -> int: - return llama_cpp.llama_n_vocab(self.model) + return llama_cpp.llama_n_vocab(self.vocab) def n_ctx_train(self) -> int: return llama_cpp.llama_n_ctx_train(self.model) @@ -84,7 +90,7 @@ def n_embd(self) -> int: return llama_cpp.llama_n_embd(self.model) def rope_freq_scale_train(self) -> float: - return llama_cpp.llama_rope_freq_scale_train(self.model) + return llama_cpp.llama_model_rope_freq_scale_train(self.model) def desc(self) -> str: buf = ctypes.create_string_buffer(1024) @@ -98,53 +104,53 @@ def n_params(self) -> int: return llama_cpp.llama_model_n_params(self.model) def get_tensor(self, name: str) -> ctypes.c_void_p: - return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) + raise NotImplementedError("get_tensor is not implemented in llama.cpp") # Vocab def token_get_text(self, token: int) -> str: - return llama_cpp.llama_token_get_text(self.model, token).decode("utf-8") + return llama_cpp.llama_token_get_text(self.vocab, token).decode("utf-8") def token_get_score(self, token: int) -> float: - return llama_cpp.llama_token_get_score(self.model, token) + return llama_cpp.llama_token_get_score(self.vocab, token) def token_get_attr(self, token: int) -> int: - return llama_cpp.llama_token_get_attr(self.model, token) + return llama_cpp.llama_token_get_attr(self.vocab, token) # Special tokens def token_bos(self) -> int: - return llama_cpp.llama_token_bos(self.model) + return llama_cpp.llama_token_bos(self.vocab) def token_eos(self) -> int: - return llama_cpp.llama_token_eos(self.model) + return llama_cpp.llama_token_eos(self.vocab) def token_cls(self) -> int: - return llama_cpp.llama_token_cls(self.model) + return llama_cpp.llama_token_cls(self.vocab) def token_sep(self) -> int: - return llama_cpp.llama_token_sep(self.model) + return llama_cpp.llama_token_sep(self.vocab) def token_nl(self) -> int: - return llama_cpp.llama_token_nl(self.model) + return llama_cpp.llama_token_nl(self.vocab) def token_prefix(self) -> int: - return llama_cpp.llama_token_prefix(self.model) + raise NotImplementedError("token_prefix is not implemented in llama.cpp") def token_middle(self) -> int: - return llama_cpp.llama_token_middle(self.model) + raise NotImplementedError("token_middle is not implemented in llama.cpp") def token_suffix(self) -> int: - return llama_cpp.llama_token_suffix(self.model) + raise NotImplementedError("token_suffix is not implemented in llama.cpp") def token_eot(self) -> int: - return llama_cpp.llama_token_eot(self.model) + return llama_cpp.llama_token_eot(self.vocab) def add_bos_token(self) -> bool: - return llama_cpp.llama_add_bos_token(self.model) + return llama_cpp.llama_add_bos_token(self.vocab) def add_eos_token(self) -> bool: - return llama_cpp.llama_add_eos_token(self.model) + return llama_cpp.llama_add_eos_token(self.vocab) # Tokenization @@ -152,13 +158,13 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): n_ctx = self.n_ctx_train() tokens = (llama_cpp.llama_token * n_ctx)() n_tokens = llama_cpp.llama_tokenize( - self.model, text, len(text), tokens, n_ctx, add_bos, special + self.vocab, text, len(text), tokens, n_ctx, add_bos, special ) if n_tokens < 0: n_tokens = abs(n_tokens) tokens = (llama_cpp.llama_token * n_tokens)() n_tokens = llama_cpp.llama_tokenize( - self.model, text, len(text), tokens, n_tokens, add_bos, special + self.vocab, text, len(text), tokens, n_tokens, add_bos, special ) if n_tokens < 0: raise RuntimeError( @@ -168,7 +174,7 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): def token_to_piece(self, token: int, special: bool = False) -> bytes: buf = ctypes.create_string_buffer(32) - llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special) + llama_cpp.llama_token_to_piece(self.vocab, token, buf, 32, 0, special) return bytes(buf) def detokenize(self, tokens: List[int], special: bool = False) -> bytes: @@ -177,7 +183,7 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: buffer = (ctypes.c_char * size)() for token in tokens: n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token), buffer, size, 0, special + self.vocab, llama_cpp.llama_token(token), buffer, size, 0, special ) assert n <= size output += bytes(buffer[:n]) @@ -320,7 +326,8 @@ def get_embeddings(self): def set_rng_seed(self, seed: int): # TODO: Fix - llama_cpp.llama_set_rng_seed(self.ctx, seed) + # llama_cpp.llama_set_rng_seed(self.ctx, seed) + raise NotImplementedError("set_rng_seed is not implemented in llama.cpp") def sample_repetition_penalties( self, @@ -331,55 +338,63 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - llama_cpp.llama_sample_repetition_penalties( - self.ctx, - llama_cpp.byref(candidates.candidates), - last_tokens_data, - penalty_last_n, - penalty_repeat, - penalty_freq, - penalty_present, - ) + # llama_cpp.llama_sample_repetition_penalties( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # last_tokens_data, + # penalty_last_n, + # penalty_repeat, + # penalty_freq, + # penalty_present, + # ) + raise NotImplementedError("sample_repetition_penalties is not implemented in llama.cpp") def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - llama_cpp.llama_sample_softmax( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + # llama_cpp.llama_sample_softmax( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) + raise NotImplementedError("sample_softmax is not implemented in llama.cpp") def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - llama_cpp.llama_sample_top_k( - self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep - ) + # llama_cpp.llama_sample_top_k( + # self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep + # ) + raise NotImplementedError("sample_top_k is not implemented in llama.cpp") def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - llama_cpp.llama_sample_top_p( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_top_p( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_top_p is not implemented in llama.cpp") def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - llama_cpp.llama_sample_min_p( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_min_p( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_min_p is not implemented in llama.cpp") def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - llama_cpp.llama_sample_typical( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_typical( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_typical is not implemented in llama.cpp") def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): - llama_cpp.llama_sample_temp( - self.ctx, llama_cpp.byref(candidates.candidates), temp - ) + # llama_cpp.llama_sample_temp( + # self.ctx, llama_cpp.byref(candidates.candidates), temp + # ) + raise NotImplementedError("sample_temp is not implemented in llama.cpp") def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - llama_cpp.llama_sample_grammar( - self.ctx, - llama_cpp.byref(candidates.candidates), - grammar.grammar, - ) + # llama_cpp.llama_sample_grammar( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # grammar.grammar, + # ) + raise NotImplementedError("sample_grammar is not implemented in llama.cpp") def sample_token_mirostat( self, @@ -389,14 +404,15 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - return llama_cpp.llama_sample_token_mirostat( - self.ctx, - llama_cpp.byref(candidates.candidates), - tau, - eta, - m, - mu, - ) + raise NotImplementedError("sample_token_mirostat is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_mirostat( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # tau, + # eta, + # m, + # mu, + # ) def sample_token_mirostat_v2( self, @@ -405,29 +421,33 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - return llama_cpp.llama_sample_token_mirostat_v2( - self.ctx, - llama_cpp.byref(candidates.candidates), - tau, - eta, - mu, - ) + raise NotImplementedError("sample_token_mirostat_v2 is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_mirostat_v2( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # tau, + # eta, + # mu, + # ) def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - return llama_cpp.llama_sample_token_greedy( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + raise NotImplementedError("sample_token_greedy is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_greedy( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - return llama_cpp.llama_sample_token( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + raise NotImplementedError("sample_token is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) + raise NotImplementedError("grammar_accept_token is not implemented in llama.cpp") + # llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -788,7 +808,7 @@ def add_mirostat_v2(self, seed: int, tau: float, eta: float): def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): sampler = llama_cpp.llama_sampler_init_grammar( - model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") + model.vocab, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") ) self._add_sampler(sampler) @@ -842,6 +862,7 @@ def get_seed(self) -> int: def sample(self, ctx: LlamaContext, idx: int) -> int: assert self.sampler is not None + assert ctx.ctx is not None return llama_cpp.llama_sampler_sample(self.sampler, ctx.ctx, idx) def close(self): diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2fd7ff193..11fc744a0 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -406,10 +406,10 @@ def __init__( ) ) - self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None + self._lora_adapter: Optional[llama_cpp.llama_adapter_lora_p] = None if self.lora_path: - self._lora_adapter = llama_cpp.llama_lora_adapter_init( + self._lora_adapter = llama_cpp.llama_adapter_lora_init( self._model.model, self.lora_path.encode("utf-8"), ) @@ -421,12 +421,12 @@ def __init__( def free_lora_adapter(): if self._lora_adapter is None: return - llama_cpp.llama_lora_adapter_free(self._lora_adapter) + llama_cpp.llama_adapter_lora_free(self._lora_adapter) self._lora_adapter = None self._stack.callback(free_lora_adapter) - if llama_cpp.llama_lora_adapter_set( + if llama_cpp.llama_set_adapter_lora( self._ctx.ctx, self._lora_adapter, self.lora_scale ): raise RuntimeError( @@ -1152,9 +1152,9 @@ def _create_completion( bos_token_id: int = self.token_bos() cls_token_id: int = self._model.token_cls() sep_token_id: int = self._model.token_sep() - prefix_token_id: int = self._model.token_prefix() - middle_token_id: int = self._model.token_middle() - suffix_token_id: int = self._model.token_suffix() + prefix_token_id: int = 0 # self._model.token_prefix() # TODO: Fix + middle_token_id: int = 0 # self._model.token_middle() # TODO: Fix + suffix_token_id: int = 0 # self._model.token_suffix() # TODO: Fix add_space_prefix: bool = ( self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" ) @@ -1332,7 +1332,7 @@ def logit_bias_processor( logits_processor=logits_processor, grammar=grammar, ): - if llama_cpp.llama_token_is_eog(self._model.model, token): + if llama_cpp.llama_token_is_eog(self._model.vocab, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" break diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 16c6b9d73..7845f0d1c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -149,6 +149,10 @@ # define LLAMA_STATE_SEQ_VERSION 2 LLAMA_STATE_SEQ_VERSION = 2 +# struct llama_vocab; +llama_vocab_p = NewType("llama_vocab_p", int) +llama_vocab_p_ctypes = ctypes.c_void_p + # struct llama_model; llama_model_p = NewType("llama_model_p", int) llama_model_p_ctypes = ctypes.c_void_p @@ -640,9 +644,6 @@ class llama_model_kv_override(ctypes.Structure): # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() # const float * tensor_split; -# // comma separated list of RPC servers to use for offloading -# const char * rpc_servers; - # // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. # // If the provided progress_callback returns true, model loading continues. # // If it returns false, model loading is immediately aborted. @@ -669,7 +670,6 @@ class llama_model_params(ctypes.Structure): split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() - rpc_servers (ctypes.c_char_p): comma separated list of RPC servers to use for offloading progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data @@ -683,7 +683,6 @@ class llama_model_params(ctypes.Structure): split_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] - rpc_servers: ctypes.c_char_p progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] @@ -698,7 +697,6 @@ class llama_model_params(ctypes.Structure): ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), - ("rpc_servers", ctypes.c_char_p), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), @@ -975,9 +973,9 @@ class llama_chat_message(ctypes.Structure): # // lora adapter -# struct llama_lora_adapter; -llama_lora_adapter_p = ctypes.c_void_p -llama_lora_adapter_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# struct llama_adapter_lora; +llama_adapter_lora_p = ctypes.c_void_p +llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p) # // Helpers for getting default parameters @@ -1059,6 +1057,18 @@ def llama_backend_init(): GGML_NUMA_STRATEGY_COUNT = 5 +# // Call once at the end of the program - currently only used for MPI +# LLAMA_API void llama_backend_free(void); +@ctypes_function( + "llama_backend_free", + [], + None, +) +def llama_backend_free(): + """Call once at the end of the program - currently only used for MPI""" + ... + + # //optional: # LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); @ctypes_function( @@ -1072,24 +1082,14 @@ def llama_numa_init(numa: int, /): # // Optional: an auto threadpool gets created in ggml if not passed explicitly # LLAMA_API void llama_attach_threadpool( -# struct llama_context * ctx, -# ggml_threadpool_t threadpool, -# ggml_threadpool_t threadpool_batch); +# struct llama_context * ctx, +# ggml_threadpool_t threadpool, +# ggml_threadpool_t threadpool_batch); +# TODO: Add llama_attach_threadpool # LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); - - -# // Call once at the end of the program - currently only used for MPI -# LLAMA_API void llama_backend_free(void); -@ctypes_function( - "llama_backend_free", - [], - None, -) -def llama_backend_free(): - """Call once at the end of the program - currently only used for MPI""" - ... +# TODO: Add llama_detach_threadpool # DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( @@ -1107,6 +1107,9 @@ def llama_load_model_from_file( ... +# // Load the model from a file +# // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf +# // If the split file name does not follow this pattern, use llama_model_load_from_splits # LLAMA_API struct llama_model * llama_model_load_from_file( # const char * path_model, # struct llama_model_params params); @@ -1118,6 +1121,31 @@ def llama_load_model_from_file( def llama_model_load_from_file( path_model: bytes, params: llama_model_params, / ) -> Optional[llama_model_p]: + """Load the model from a file + + If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + + If the split file name does not follow this pattern, use llama_model_load_from_splits""" + ... + + +# // Load the model from multiple splits (support custom naming scheme) +# // The paths must be in the correct order +# LLAMA_API struct llama_model * llama_model_load_from_splits( +# const char ** paths, +# size_t n_paths, +# struct llama_model_params params); +@ctypes_function( + "llama_model_load_from_splits", + [ctypes.POINTER(ctypes.c_char_p), ctypes.c_size_t, llama_model_params], + llama_model_p_ctypes, +) +def llama_model_load_from_splits( + paths: List[bytes], n_paths: int, params: llama_model_params, / +) -> Optional[llama_model_p]: + """Load the model from multiple splits (support custom naming scheme) + + The paths must be in the correct order""" ... @@ -1141,9 +1169,24 @@ def llama_model_free(model: llama_model_p, /): ... -# LLAMA_API struct llama_context * llama_new_context_with_model( +# LLAMA_API struct llama_context * llama_init_from_model( # struct llama_model * model, # struct llama_context_params params); +@ctypes_function( + "llama_init_from_model", + [llama_model_p_ctypes, llama_context_params], + llama_context_p_ctypes, +) +def llama_init_from_model( + model: llama_model_p, params: llama_context_params, / +) -> Optional[llama_context_p]: + ... + + +# DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( +# struct llama_model * model, +# struct llama_context_params params), +# "use llama_init_from_model instead"); @ctypes_function( "llama_new_context_with_model", [llama_model_p_ctypes, llama_context_params], @@ -1231,65 +1274,102 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... -# LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); -@ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... -# LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_ctx_train(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_embd (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_embd(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_layer (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_layer(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_head (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); @ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_head(model: llama_model_p, /) -> int: ... -# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); +# DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); +@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32) +def llama_n_vocab(model: llama_vocab_p, /) -> int: + ... + + +# LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) def llama_pooling_type(ctx: llama_context_p, /) -> int: ... -# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); -@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: +# LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); +@ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes) +def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: + ... + + +# LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); +@ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_rope_type(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); +@ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_ctx_train(model: llama_model_p, /) -> int: ... -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); -@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: +# LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); +@ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_embd(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); +@ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_layer(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); +@ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_head(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor -# LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model); -@ctypes_function("llama_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) -def llama_rope_freq_scale_train(model: llama_model_p, /) -> float: - """Get the model's RoPE frequency scaling factor""" +# LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); +@ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) +def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: + ... + + +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); +@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_vocab_type(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab); +@ctypes_function("llama_vocab_n_tokens", [llama_vocab_p_ctypes], ctypes.c_int32) +def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int: ... @@ -1402,6 +1482,14 @@ def llama_model_size(model: llama_model_p, /) -> int: ... +# // Get the default chat template. Returns nullptr if not available +# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model); +@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes], ctypes.c_char_p) +def llama_model_chat_template(model: llama_model_p, /) -> Optional[bytes]: + """Get the default chat template. Returns None if not available""" + ... + + # // Returns the total number of parameters in the model # LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model); @ctypes_function("llama_model_n_params", [llama_model_p_ctypes], ctypes.c_uint64) @@ -1472,37 +1560,48 @@ def llama_model_quantize( # // Load a LoRA adapter from file -# // The loaded adapter will be associated to the given model, and will be free when the model is deleted -# LLAMA_API struct llama_lora_adapter * llama_lora_adapter_init( +# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init( # struct llama_model * model, # const char * path_lora); @ctypes_function( - "llama_lora_adapter_init", + "llama_adapter_lora_init", [llama_model_p_ctypes, ctypes.c_char_p], - llama_lora_adapter_p_ctypes, + llama_adapter_lora_p_ctypes, ) -def llama_lora_adapter_init( +def llama_adapter_lora_init( model: llama_model_p, path_lora: bytes, / -) -> Optional[llama_lora_adapter_p]: - """Load a LoRA adapter from file - The loaded adapter will be associated to the given model, and will be free when the model is deleted - """ +) -> Optional[llama_adapter_lora_p]: + ... + + +# // Manually free a LoRA adapter +# // Note: loaded adapters will be free when the associated model is deleted +# LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter); +@ctypes_function( + "llama_adapter_lora_free", + [llama_adapter_lora_p_ctypes], + None, +) +def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): ... +# // The following functions operate on a llama_context, hence the naming: llama_verb_... + + # // Add a loaded LoRA adapter to given context # // This will not modify model's weight -# LLAMA_API int32_t llama_lora_adapter_set( +# LLAMA_API int32_t llama_set_adapter_lora( # struct llama_context * ctx, -# struct llama_lora_adapter * adapter, +# struct llama_adapter_lora * adapter, # float scale); @ctypes_function( - "llama_lora_adapter_set", - [llama_context_p_ctypes, llama_lora_adapter_p_ctypes, ctypes.c_float], + "llama_set_adapter_lora", + [llama_context_p_ctypes, llama_adapter_lora_p_ctypes, ctypes.c_float], ctypes.c_int32, ) -def llama_lora_adapter_set( - ctx: llama_context_p, adapter: llama_lora_adapter_p, scale: float, / +def llama_set_adapter_lora( + ctx: llama_context_p, adapter: llama_adapter_lora_p, scale: float, / ) -> int: """Add a loaded LoRA adapter to given context This will not modify model's weight""" @@ -1511,64 +1610,49 @@ def llama_lora_adapter_set( # // Remove a specific LoRA adapter from given context # // Return -1 if the adapter is not present in the context -# LLAMA_API int32_t llama_lora_adapter_remove( +# LLAMA_API int32_t llama_rm_adapter_lora( # struct llama_context * ctx, -# struct llama_lora_adapter * adapter); +# struct llama_adapter_lora * adapter); @ctypes_function( - "llama_lora_adapter_remove", - [llama_context_p_ctypes, llama_lora_adapter_p_ctypes], + "llama_rm_adapter_lora", + [llama_context_p_ctypes, llama_adapter_lora_p_ctypes], ctypes.c_int32, ) -def llama_lora_adapter_remove( - ctx: llama_context_p, adapter: llama_lora_adapter_p, / +def llama_rm_adapter_lora( + ctx: llama_context_p, adapter: llama_adapter_lora_p, / ) -> int: - """Remove a LoRA adapter from given context + """Remove a specific LoRA adapter from given context Return -1 if the adapter is not present in the context""" ... # // Remove all LoRA adapters from given context -# LLAMA_API void llama_lora_adapter_clear( -# struct llama_context * ctx); +# LLAMA_API void llama_clear_adapter_lora(struct llama_context * ctx); @ctypes_function( - "llama_lora_adapter_clear", + "llama_clear_adapter_lora", [llama_context_p_ctypes], None, ) -def llama_lora_adapter_clear(ctx: llama_context_p, /): +def llama_clear_adapter_lora(ctx: llama_context_p, /): """Remove all LoRA adapters from given context""" ... -# // Manually free a LoRA adapter -# // Note: loaded adapters will be free when the associated model is deleted -# LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); -@ctypes_function( - "llama_lora_adapter_free", - [llama_lora_adapter_p_ctypes], - None, -) -def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): - """Manually free a LoRA adapter - Note: loaded adapters will be free when the associated model is deleted""" - ... - - # // Apply a loaded control vector to a llama_context, or if data is NULL, clear # // the currently loaded vector. # // n_embd should be the size of a single layer's control, and data should point # // to an n_embd x n_layers buffer starting from layer 1. # // il_start and il_end are the layer range the vector should apply to (both inclusive) # // See llama_control_vector_load in common to load a control vector. -# LLAMA_API int32_t llama_control_vector_apply( -# struct llama_context * lctx, +# LLAMA_API int32_t llama_apply_adapter_cvec( +# struct llama_context * ctx, # const float * data, # size_t len, # int32_t n_embd, # int32_t il_start, # int32_t il_end); @ctypes_function( - "llama_control_vector_apply", + "llama_apply_adapter_cvec", [ llama_context_p_ctypes, ctypes.POINTER(ctypes.c_float), @@ -1579,8 +1663,8 @@ def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): ], ctypes.c_int32, ) -def llama_control_vector_apply( - lctx: llama_context_p, +def llama_apply_adapter_cvec( + ctx: llama_context_p, data: CtypesPointerOrRef[ctypes.c_float], len: int, n_embd: int, @@ -2577,53 +2661,53 @@ def llama_get_embeddings_seq( # // -# LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token); +# LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_text", [llama_model_p_ctypes, llama_token], ctypes.c_char_p + "llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p ) -def llama_token_get_text( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_text( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bytes: ... -# LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); +# LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_score", [llama_model_p_ctypes, llama_token], ctypes.c_float + "llama_vocab_get_score", [llama_vocab_p_ctypes, llama_token], ctypes.c_float ) -def llama_token_get_score( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_score( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> float: ... -# LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); +# LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_attr", [llama_model_p_ctypes, llama_token], ctypes.c_int + "llama_vocab_get_attr", [llama_vocab_p_ctypes, llama_token], ctypes.c_int ) -def llama_token_get_attr( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_attr( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) -# LLAMA_API bool llama_token_is_eog(const struct llama_model * model, llama_token token); +# LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_is_eog", [llama_model_p_ctypes, llama_token], ctypes.c_bool + "llama_vocab_is_eog", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) -> bool: +def llama_vocab_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool: """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)""" ... # // Identify if Token Id is a control token or a render-able token -# LLAMA_API bool llama_token_is_control(const struct llama_model * model, llama_token token); +# LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_is_control", [llama_model_p_ctypes, llama_token], ctypes.c_bool + "llama_vocab_is_control", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_control( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_is_control( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bool: """Identify if Token Id is a control token or a render-able token""" ... @@ -2632,110 +2716,335 @@ def llama_token_is_control( # // Special tokens -# LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence -@ctypes_function("llama_token_bos", [llama_model_p_ctypes], llama_token) -def llama_token_bos(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence +@ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token: """beginning-of-sentence""" ... -# LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence -@ctypes_function("llama_token_eos", [llama_model_p_ctypes], llama_token) -def llama_token_eos(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence +@ctypes_function("llama_vocab_eos", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_eos(vocab: llama_vocab_p, /) -> llama_token: """end-of-sentence""" ... -# LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn -@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn +@ctypes_function("llama_vocab_eot", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_eot(vocab: llama_vocab_p, /) -> llama_token: """end-of-turn""" ... -# LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification -@ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token) -def llama_token_cls(model: llama_model_p, /) -> int: - """classification""" +# LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator +@ctypes_function("llama_vocab_sep", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_sep(vocab: llama_vocab_p, /) -> llama_token: + """sentence separator""" ... -# LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator -@ctypes_function("llama_token_sep", [llama_model_p_ctypes], llama_token) -def llama_token_sep(model: llama_model_p, /) -> int: - """sentence separator""" +# LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line +@ctypes_function("llama_vocab_nl", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_nl(vocab: llama_vocab_p, /) -> llama_token: + """next-line""" ... -# LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line -@ctypes_function("llama_token_nl", [llama_model_p_ctypes], llama_token) -def llama_token_nl(model: llama_model_p, /) -> int: - """next-line""" +# LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding +@ctypes_function("llama_vocab_pad", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token: + """padding""" + ... + +# LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_get_add_bos", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: ... -# LLAMA_API bool llama_add_bos_token(const struct llama_model * model); -@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: +# LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_get_add_eos", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... -# LLAMA_API bool llama_add_eos_token(const struct llama_model * model); -@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: +# LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_pre", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... -# // Codellama infill tokens -# DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead"); -@ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token) -def llama_token_prefix(model: llama_model_p) -> int: - """codellama infill tokens""" +# LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_suf", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... -# DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead"); -@ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_mid", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... -# DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead"); -@ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model); -@ctypes_function("llama_token_fim_pre", [llama_model_p_ctypes], llama_token) -def llama_token_fim_pre(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_rep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model); -@ctypes_function("llama_token_fim_suf", [llama_model_p_ctypes], llama_token) -def llama_token_fim_suf(model: llama_model_p, /) -> int: + +# LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model); -@ctypes_function("llama_token_fim_mid", [llama_model_p_ctypes], llama_token) -def llama_token_fim_mid(model: llama_model_p, /) -> int: + + +# DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); +@ctypes_function( + "llama_token_get_text", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_char_p, +) +def llama_token_get_text( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bytes: + ... + + +# DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); +@ctypes_function( + "llama_token_get_score", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_float, +) +def llama_token_get_score( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> float: + ... + +# DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); +@ctypes_function( + "llama_token_get_attr", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_int, +) +def llama_token_get_attr( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> int: + ... + +# DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); +@ctypes_function( + "llama_token_is_eog", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_bool, +) +def llama_token_is_eog( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bool: + ... + +# DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); +@ctypes_function( + "llama_token_is_control", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_bool, +) +def llama_token_is_control( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bool: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); +@ctypes_function( + "llama_token_bos", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_bos(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); +@ctypes_function( + "llama_token_eos", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_eos(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); +@ctypes_function( + "llama_token_eot", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_eot(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); +@ctypes_function( + "llama_token_cls", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_cls(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); +@ctypes_function( + "llama_token_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_sep(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); +@ctypes_function( + "llama_token_nl", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_nl(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); +@ctypes_function( + "llama_token_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_pad(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); +@ctypes_function( + "llama_add_bos_token", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: + ... + +# DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); +@ctypes_function( + "llama_add_eos_token", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); +@ctypes_function( + "llama_token_fim_pre", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model); -@ctypes_function("llama_token_fim_pad", [llama_model_p_ctypes], llama_token) -def llama_token_fim_pad(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); +@ctypes_function( + "llama_token_fim_suf", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model); -@ctypes_function("llama_token_fim_rep", [llama_model_p_ctypes], llama_token) -def llama_token_fim_rep(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); +@ctypes_function( + "llama_token_fim_mid", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model); -@ctypes_function("llama_token_fim_sep", [llama_model_p_ctypes], llama_token) -def llama_token_fim_sep(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); +@ctypes_function( + "llama_token_fim_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... +# DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); +@ctypes_function( + "llama_token_fim_rep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); +@ctypes_function( + "llama_token_fim_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: + ... + +# // CLS is equivalent to BOS +# DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification +# "use llama_vocab_bos instead"); +@ctypes_function( + "llama_vocab_cls", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: + ... + + # // # // Tokenization # // @@ -2751,7 +3060,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: # /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated # /// as plaintext. Does not insert a leading space. # LLAMA_API int32_t llama_tokenize( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # const char * text, # int32_t text_len, # llama_token * tokens, @@ -2761,7 +3070,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: @ctypes_function( "llama_tokenize", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_int32, llama_token_p, @@ -2772,7 +3081,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: ctypes.c_int32, ) def llama_tokenize( - model: llama_model_p, + vocab: llama_vocab_p, text: bytes, text_len: Union[ctypes.c_int, int], tokens: CtypesArray[llama_token], @@ -2784,7 +3093,7 @@ def llama_tokenize( """Convert the provided text into tokens. Args: - model: The model to use for tokenization. + vocab: The vocabulary to use for tokenization. text: The text to tokenize. text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. @@ -2805,7 +3114,7 @@ def llama_tokenize( # // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') # // @param special If true, special tokens are rendered in the output. # LLAMA_API int32_t llama_token_to_piece( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # llama_token token, # char * buf, # int32_t length, @@ -2814,7 +3123,7 @@ def llama_tokenize( @ctypes_function( "llama_token_to_piece", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, llama_token, ctypes.c_char_p, ctypes.c_int32, @@ -2824,7 +3133,7 @@ def llama_tokenize( ctypes.c_int32, ) def llama_token_to_piece( - model: llama_model_p, + vocab: llama_vocab_p, token: Union[llama_token, int], buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]], length: Union[ctypes.c_int, int], @@ -2838,7 +3147,7 @@ def llama_token_to_piece( User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. Args: - model: The model to use for tokenization. + vocab: The vocabulary to use for tokenization. token: The token to convert. buf: The buffer to write the token to. length: The length of the buffer. @@ -2930,7 +3239,6 @@ def llama_detokenize( # /// @param length The size of the allocated buffer # /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. # LLAMA_API int32_t llama_chat_apply_template( -# const struct llama_model * model, # const char * tmpl, # const struct llama_chat_message * chat, # size_t n_msg, @@ -2940,20 +3248,37 @@ def llama_detokenize( @ctypes_function( "llama_chat_apply_template", [ - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.POINTER(llama_chat_message), - ctypes.c_size_t, + ctypes.c_char_p, # tmpl + ctypes.POINTER(llama_chat_message), # chat + ctypes.c_size_t, # n_msg + ctypes.c_bool, # add_ass (added) + ctypes.c_char_p, # buf + ctypes.c_int32, # length ], ctypes.c_int32, ) def llama_chat_apply_template( - model: llama_model_p, tmpl: bytes, chat: CtypesArray[llama_chat_message], n_msg: int, + add_ass: bool, # Added parameter + buf: bytes, + length: int, /, ) -> int: + """Apply chat template. + + Args: + tmpl: Template to use. If None, uses model's default + chat: Array of chat messages + n_msg: Number of messages + add_ass: Whether to end prompt with assistant token + buf: Output buffer + length: Buffer length + + Returns: + Number of bytes written, or needed if buffer too small + """ ... @@ -3022,7 +3347,6 @@ def llama_chat_builtin_templates( # // llama_sampler_free(smpl); # // # // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU). -# // TODO: in the future, the entire sampling API that uses llama_model should start using llama_vocab # // # typedef void * llama_sampler_context_t; @@ -3342,16 +3666,16 @@ def llama_sampler_init_mirostat_v2( # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # const char * grammar_str, # const char * grammar_root); @ctypes_function( "llama_sampler_init_grammar", - [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], + [llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], llama_sampler_p_ctypes, ) def llama_sampler_init_grammar( - model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / + vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, / ) -> llama_sampler_p: ... @@ -3379,7 +3703,8 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 # LLAMA_API struct llama_sampler * llama_sampler_init_dry( -# const struct llama_model * model, +# const struct llama_vocab * vocab, +# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, @@ -3389,7 +3714,8 @@ def llama_sampler_init_penalties( @ctypes_function( "llama_sampler_init_dry", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, + ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -3400,7 +3726,8 @@ def llama_sampler_init_penalties( llama_sampler_p_ctypes, ) def llama_sampler_init_dry( - model: llama_model_p, + vocab: llama_vocab_p, + n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -3448,15 +3775,13 @@ def llama_sampler_init_logit_bias( # // 3. discard non-EOG tokens with low prob # // 4. if no tokens are left -> pick EOT # // -# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_model * model); +# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); @ctypes_function( "llama_sampler_init_infill", - [llama_model_p_ctypes], + [llama_vocab_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_init_infill(model: llama_model_p, /) -> llama_sampler_p: - """This sampler is meant to be used for fill-in-the-middle infilling. - """ +def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f7cd13301..ef6dada60 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f7cd13301c2a88f97073fd119072b4cc92c08df1 +Subproject commit ef6dada60ca710f4edfbb6fd9e1258685d8ea49d From 0b89fe48ad26ffcff76451bd87642d916a1a3385 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 21:40:53 -0500 Subject: [PATCH 145/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 ++++++---- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 7845f0d1c..2b9edbb99 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1483,10 +1483,12 @@ def llama_model_size(model: llama_model_p, /) -> int: # // Get the default chat template. Returns nullptr if not available -# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model); -@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes], ctypes.c_char_p) -def llama_model_chat_template(model: llama_model_p, /) -> Optional[bytes]: - """Get the default chat template. Returns None if not available""" +# // If name is NULL, returns the default chat template +# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name); +@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_char_p) +def llama_model_chat_template(model: llama_model_p, name: Optional[bytes], /) -> Optional[bytes]: + """Get the default chat template. Returns None if not available + If name is None, returns the default chat template""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ef6dada60..794fe23f2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ef6dada60ca710f4edfbb6fd9e1258685d8ea49d +Subproject commit 794fe23f29fb40104975c91fe19f23798f7c726e From 14879c7819eb023a6a272e0f18aca887ae605c0f Mon Sep 17 00:00:00 2001 From: oobabooga Date: Tue, 28 Jan 2025 23:43:29 -0300 Subject: [PATCH 146/155] fix(ci): Fix the CUDA workflow (#1894) --- .github/workflows/build-wheels-cuda.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3d410148f..745b2e602 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -61,11 +61,9 @@ jobs: - name: Setup Mamba uses: conda-incubator/setup-miniconda@v3.1.0 with: - activate-environment: "build" + activate-environment: "llamacpp" python-version: ${{ matrix.pyver }} - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true add-pip-as-python-dependency: true auto-activate-base: false From 4442ff8e0a09feafcf7c91045ebbe0ba6f026feb Mon Sep 17 00:00:00 2001 From: Shaka Huang Date: Wed, 29 Jan 2025 11:57:37 +0800 Subject: [PATCH 147/155] fix: error showing time spent in llama perf context print (#1898) * feat: Sync with llama.cpp Add `no_perf` field to `llama_context_params` to optionally disable performance timing measurements. * fix: Display performance metrics by default --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 4 ++++ llama_cpp/llama_cpp.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 11fc744a0..7e9a6af23 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -94,6 +94,7 @@ def __init__( offload_kqv: bool = True, flash_attn: bool = False, # Sampling Params + no_perf: bool = False, last_n_tokens_size: int = 64, # LoRA Params lora_base: Optional[str] = None, @@ -173,6 +174,7 @@ def __init__( embedding: Embedding mode only. offload_kqv: Offload K, Q, V to GPU. flash_attn: Use flash attention. + no_perf: Measure performance timings. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. lora_path: Path to a LoRA file to apply to the model. @@ -351,6 +353,7 @@ def __init__( if type_v is not None: self.context_params.type_v = type_v # Sampling Params + self.context_params.no_perf = no_perf self.last_n_tokens_size = last_n_tokens_size self.cache: Optional[BaseLlamaCache] = None @@ -2093,6 +2096,7 @@ def __getstate__(self): offload_kqv=self.context_params.offload_kqv, flash_attn=self.context_params.flash_attn, # Sampling Params + no_perf=self.context_params.no_perf, last_n_tokens_size=self.last_n_tokens_size, # LoRA Params lora_base=self.lora_base, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2b9edbb99..205d89a0b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -780,6 +780,7 @@ class llama_context_params(ctypes.Structure): embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention + no_perf (bool): whether to measure performance timings abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ @@ -810,6 +811,7 @@ class llama_context_params(ctypes.Structure): embeddings: bool offload_kqv: bool flash_attn: bool + no_perf: bool abort_callback: Callable[[ctypes.c_void_p], bool] abort_callback_data: ctypes.c_void_p @@ -839,6 +841,7 @@ class llama_context_params(ctypes.Structure): ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), ("flash_attn", ctypes.c_bool), + ("no_perf", ctypes.c_bool), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), ] From 710e19a81284e5af0d5db93cef7a9063b3e8534f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 23:19:04 -0500 Subject: [PATCH 148/155] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f49c904..53365e368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.7] + +- feat: Update llama.cpp to ggerganov/llama.cpp@794fe23f29fb40104975c91fe19f23798f7c726e +- fix(ci): Fix the CUDA workflow by @oobabooga in #1894 +- fix: error showing time spent in llama perf context print, adds `no_perf` flag to `Llama` class by @shakalaca in #1898 + ## [0.3.6] - feat: Update llama.cpp to ggerganov/llama.cpp@f7cd13301c2a88f97073fd119072b4cc92c08df1 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 8c6118fb4..fc1fcbcf6 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.6" +__version__ = "0.3.7" From 344c106a6961b50f734b95f084fbf02057d4b475 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 04:11:42 -0400 Subject: [PATCH 149/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 74 ++++++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 205d89a0b..f3985ad2f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -227,6 +227,7 @@ # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -257,6 +258,7 @@ LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 +LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 # // note: these values should be synchronized with ggml_rope @@ -1357,6 +1359,12 @@ def llama_model_n_head(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); +@ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_head_kv(model: llama_model_p, /) -> int: + ... + + # // Get the model's RoPE frequency scaling factor # LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); @ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) @@ -3375,8 +3383,8 @@ class llama_sampler_i(ctypes.Structure): # struct llama_sampler { -# struct llama_sampler_i * iface; -# llama_sampler_context_t ctx; +# const struct llama_sampler_i * iface; +# llama_sampler_context_t ctx; # }; class llama_sampler(ctypes.Structure): _fields_ = [ @@ -3410,6 +3418,18 @@ class llama_sampler(ctypes.Structure): # // mirror of llama_sampler_i: +# LLAMA_API struct llama_sampler * llama_sampler_init (const struct llama_sampler_i * iface, llama_sampler_context_t ctx); +@ctypes_function( + "llama_sampler_init", + [ctypes.POINTER(llama_sampler_i), llama_sampler_context_t], + llama_sampler_p_ctypes, +) +def llama_sampler_init( + iface: ctypes.POINTER(llama_sampler_i), ctx: llama_sampler_context_t, / +) -> llama_sampler_p: + ... + + # LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); @ctypes_function( "llama_sampler_name", @@ -3627,6 +3647,17 @@ def llama_sampler_init_xtc( ... +# /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 +# LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); +@ctypes_function( + "llama_sampler_init_top_n_sigma", + [ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: + ... + + # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3685,6 +3716,43 @@ def llama_sampler_init_grammar( ... +# /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 +# /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. +# /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. +# LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( +# const struct llama_vocab * vocab, +# const char * grammar_str, +# const char * grammar_root, +# const char ** trigger_patterns, +# size_t num_trigger_patterns, +# const llama_token * trigger_tokens, +# size_t num_trigger_tokens); +@ctypes_function( + "llama_sampler_init_grammar_lazy_patterns", + [ + llama_vocab_p_ctypes, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ctypes.POINTER(llama_token), + ctypes.c_size_t, + ], + llama_sampler_p_ctypes, +) +def llama_sampler_init_grammar_lazy_patterns( + vocab: llama_vocab_p, + grammar_str: bytes, + grammar_root: bytes, + trigger_patterns: CtypesArray[bytes], + num_trigger_patterns: int, + trigger_tokens: CtypesArray[llama_token], + num_trigger_tokens: int, + /, +) -> llama_sampler_p: + ... + + # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) @@ -3737,7 +3805,7 @@ def llama_sampler_init_dry( dry_base: float, dry_allowed_length: int, dry_penalty_last_n: int, - seq_breakers: CtypesArray[bytes], + seq_breakers, num_breakers: int, /, ) -> llama_sampler_p: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 794fe23f2..2c9f833d1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 794fe23f29fb40104975c91fe19f23798f7c726e +Subproject commit 2c9f833d17bb5b8ea89dec663b072b5420fc5438 From e232fae1a35c0bdaabf230a5d71780c0178e5d8f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 04:44:18 -0400 Subject: [PATCH 150/155] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 2c9f833d1..7841fc723 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 2c9f833d17bb5b8ea89dec663b072b5420fc5438 +Subproject commit 7841fc723e059d1fd9640e5c0ef19050fcc7c698 From 37eb5f0a4c2a8706b89ead1406b1577c4602cdec Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 05:30:21 -0400 Subject: [PATCH 151/155] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53365e368..605370e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.8] + +- feat: Update llama.cpp to ggerganov/llama.cpp@7841fc723e059d1fd9640e5c0ef19050fcc7c698 + ## [0.3.7] - feat: Update llama.cpp to ggerganov/llama.cpp@794fe23f29fb40104975c91fe19f23798f7c726e diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fc1fcbcf6..b1a8b9baa 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.7" +__version__ = "0.3.8" From 99f2ebfde18912adeb7f714b49c1ddb624df3087 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 11 Apr 2025 03:28:20 -0400 Subject: [PATCH 152/155] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 268 ++++++++++++++++++++++++++++++++++++++--- vendor/llama.cpp | 2 +- 2 files changed, 255 insertions(+), 15 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index f3985ad2f..710bd83c8 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -165,6 +165,10 @@ # llama_sampler_p = NewType("llama_sampler_p", int) # llama_sampler_p_ctypes = ctypes.c_void_p +# struct llama_kv_cache; +llama_kv_cache_p = NewType("llama_kv_cache_p", int) +llama_kv_cache_p_ctypes = ctypes.c_void_p + # typedef int32_t llama_pos; llama_pos = ctypes.c_int32 # typedef int32_t llama_token; @@ -228,6 +232,9 @@ # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, # LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -259,6 +266,9 @@ LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 +LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 +LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 +LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 # // note: these values should be synchronized with ggml_rope @@ -630,10 +640,19 @@ class llama_model_kv_override(ctypes.Structure): value: Union[int, float, bool, bytes] +# struct llama_model_tensor_buft_override { +# const char * pattern; +# ggml_backend_buffer_type_t buft; +# }; + + # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; +# // NULL-terminated list of buffer types to use for tensors that match a pattern +# const struct llama_model_tensor_buft_override * tensor_buft_overrides; + # int32_t n_gpu_layers; // number of layers to store in VRAM # enum llama_split_mode split_mode; // how to split the model across multiple GPUs @@ -668,6 +687,8 @@ class llama_model_params(ctypes.Structure): """Parameters for llama_model Attributes: + devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored @@ -681,6 +702,8 @@ class llama_model_params(ctypes.Structure): check_tensors (bool): validate model tensor data""" if TYPE_CHECKING: + devices: CtypesArray[ctypes.c_void_p] # NOTE: unused + tensor_buft_overrides: CtypesArray[llama_model_tensor_buft_override] # NOTE: unused n_gpu_layers: int split_mode: int main_gpu: int @@ -695,6 +718,7 @@ class llama_model_params(ctypes.Structure): _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused + ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), @@ -1317,7 +1341,18 @@ def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +# LLAMA_API struct llama_kv_cache * llama_get_kv_self ( struct llama_context * ctx); +@ctypes_function( + "llama_get_kv_self", + [llama_context_p_ctypes], + llama_kv_cache_p_ctypes, +) +def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: + """Get the KV cache for self-attention""" + ... + + +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) def llama_pooling_type(ctx: llama_context_p, /) -> int: ... @@ -1810,7 +1845,19 @@ def llama_kv_cache_view_update(ctx: llama_context_p, view: CtypesPointerOrRef[ll # // Returns the number of tokens in the KV cache (slow, use only for debug) # // If a KV cell has multiple sequences assigned to it, it will be counted multiple times -# LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx); +# LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx); +@ctypes_function( + "llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32 +) +def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: + """Returns the number of tokens in the KV cache (slow, use only for debug) + If a KV cell has multiple sequences assigned to it, it will be counted multiple times + """ + ... + + +# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx), +# "use llama_kv_self_n_tokens instead"); @ctypes_function( "llama_get_kv_cache_token_count", [llama_context_p_ctypes], ctypes.c_int32 ) @@ -1822,7 +1869,17 @@ def llama_get_kv_cache_token_count(ctx: llama_context_p, /) -> int: # // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) -# LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx); +# LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx); +@ctypes_function( + "llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32 +) +def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: + """Returns the number of used KV cells (i.e. have at least one sequence assigned to them)""" + ... + + +# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx), +# "use llama_kv_self_used_cells instead"); @ctypes_function( "llama_get_kv_cache_used_cells", [llama_context_p_ctypes], ctypes.c_int32 ) @@ -1832,9 +1889,17 @@ def llama_get_kv_cache_used_cells(ctx: llama_context_p, /) -> int: # // Clear the KV cache - both cell info is erased and KV data is zeroed -# LLAMA_API void llama_kv_cache_clear( +# LLAMA_API void llama_kv_self_clear( # struct llama_context * ctx); -@ctypes_function("llama_kv_cache_clear", [llama_context_p_ctypes], None) +@ctypes_function( + "llama_kv_self_clear", [llama_context_p_ctypes], None +) +def llama_kv_self_clear(ctx: llama_context_p, /): + """Clear the KV cache - both cell info is erased and KV data is zeroed""" + ... + +# NOTE: Deprecated +@ctypes_function("llama_kv_self_clear", [llama_context_p_ctypes], None) def llama_kv_cache_clear(ctx: llama_context_p, /): """Clear the KV cache""" ... @@ -1881,14 +1946,41 @@ def llama_kv_cache_seq_rm( # // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_cp( +# LLAMA_API void llama_kv_self_seq_cp( # struct llama_context * ctx, # llama_seq_id seq_id_src, # llama_seq_id seq_id_dst, # llama_pos p0, # llama_pos p1); @ctypes_function( - "llama_kv_cache_seq_cp", + "llama_kv_self_seq_cp", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_seq_id, + llama_pos, + llama_pos, + ], + None, +) +def llama_kv_self_seq_cp( + ctx: llama_context_p, + seq_id_src: Union[llama_seq_id, int], + seq_id_dst: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + /, +): + """Copy all tokens that belong to the specified sequence to another sequence + Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# NOTE: Deprecated +@ctypes_function( + "llama_kv_self_seq_cp", [ llama_context_p_ctypes, llama_seq_id, @@ -1914,17 +2006,27 @@ def llama_kv_cache_seq_cp( # // Removes all tokens that do not belong to the specified sequence -# LLAMA_API void llama_kv_cache_seq_keep( +# LLAMA_API void llama_kv_self_seq_keep( # struct llama_context * ctx, # llama_seq_id seq_id); @ctypes_function( - "llama_kv_cache_seq_keep", [llama_context_p_ctypes, llama_seq_id], None + "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None +) +def llama_kv_self_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): + """Removes all tokens that do not belong to the specified sequence""" + ... + + +# NOTE: Deprecated +@ctypes_function( + "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None ) def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): """Removes all tokens that do not belong to the specified sequence""" ... + # // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) # // If the KV cache is RoPEd, the KV data is updated accordingly: # // - lazily on next llama_decode() @@ -1938,7 +2040,48 @@ def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, in # llama_pos p1, # llama_pos delta); @ctypes_function( - "llama_kv_cache_seq_add", + "llama_kv_self_seq_add", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + llama_pos, + ], + None, +) +def llama_kv_self_seq_add( + ctx: llama_context_p, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + delta: Union[llama_pos, int], + /, +): + """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) + If the KV cache is RoPEd, the KV data is updated accordingly: + - lazily on next llama_decode() + - explicitly with llama_kv_cache_update() + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# // NOTE: Deprecated +# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) +# // If the KV cache is RoPEd, the KV data is updated accordingly: +# // - lazily on next llama_decode() +# // - explicitly with llama_kv_cache_update() +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_kv_cache_seq_add( +# struct llama_context * ctx, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# llama_pos delta); +@ctypes_function( + "llama_kv_self_seq_add", [ llama_context_p_ctypes, llama_seq_id, @@ -1976,7 +2119,44 @@ def llama_kv_cache_seq_add( # llama_pos p1, # int d); @ctypes_function( - "llama_kv_cache_seq_div", + "llama_kv_self_seq_div", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + ctypes.c_int, + ], + None, +) +def llama_kv_self_seq_div( + ctx: llama_context_p, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + d: Union[ctypes.c_int, int], + /, +): + """Integer division of the positions by factor of `d > 1` + If the KV cache is RoPEd, the KV data is updated accordingly + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# // NOTE: Deprecated +# // Integer division of the positions by factor of `d > 1` +# // If the KV cache is RoPEd, the KV data is updated accordingly +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_kv_cache_seq_div( +# struct llama_context * ctx, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# int d); +@ctypes_function( + "llama_kv_self_seq_div", [ llama_context_p_ctypes, llama_seq_id, @@ -2001,10 +2181,39 @@ def llama_kv_cache_seq_div( ... +# // Returns the largest position present in the KV cache for the specified sequence +# LLAMA_API llama_pos llama_kv_self_seq_pos_max( +# struct llama_context * ctx, +# llama_seq_id seq_id); +@ctypes_function( + "llama_kv_self_seq_pos_max", [llama_context_p_ctypes, llama_seq_id], llama_pos +) +def llama_kv_self_seq_pos_max( + ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / +) -> int: + """Returns the largest position present in the KV cache for the specified sequence""" + ... + + # // Defragment the KV cache # // This will be applied: # // - lazily on next llama_decode() -# // - explicitly with llama_kv_cache_update() +# // - explicitly with llama_kv_self_update() +# LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx); +@ctypes_function("llama_kv_self_defrag", [llama_context_p_ctypes], None) +def llama_kv_self_defrag(ctx: llama_context_p, /): + """Defragment the KV cache + This will be applied: + - lazily on next llama_decode() + - explicitly with llama_kv_cache_update()""" + ... + + +# NOTE: Deprecated +# // Defragment the KV cache +# // This will be applied: +# // - lazily on next llama_decode() +# // - explicitly with llama_kv_self_update() # LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx); @ctypes_function("llama_kv_cache_defrag", [llama_context_p_ctypes], None) def llama_kv_cache_defrag(ctx: llama_context_p, /): @@ -2017,7 +2226,15 @@ def llama_kv_cache_defrag(ctx: llama_context_p, /): # // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) # LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); -@ctypes_function("llama_kv_cache_update", [llama_context_p_ctypes], None) +@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) +def llama_kv_self_update(ctx: llama_context_p, /): + """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" + ... + +# // NOTE: Deprecated +# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) +# LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); +@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) def llama_kv_cache_update(ctx: llama_context_p, /): """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" ... @@ -2025,7 +2242,16 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // Check if the context supports KV cache shifting # LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); -@ctypes_function("llama_kv_cache_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +def llama_kv_self_can_shift(ctx: llama_context_p, /) -> bool: + """Check if the context supports KV cache shifting""" + ... + + +# // NOTE: Deprecated +# // Check if the context supports KV cache shifting +# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); +@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: """Check if the context supports KV cache shifting""" ... @@ -2547,6 +2773,16 @@ def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): ... +# // Set whether the model is in warmup mode or not +# // If true, all model tensors are activated during llama_decode() to load and cache their weights. +# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); +@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_warmup(ctx: llama_context_p, warmup: bool, /): + """Set whether the model is in warmup mode or not + If true, all model tensors are activated during llama_decode() to load and cache their weights.""" + ... + + # // Set abort callback # LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); @ctypes_function( @@ -3701,6 +3937,10 @@ def llama_sampler_init_mirostat_v2( ... +# /// @details Intializes a GBNF grammar, see grammars/README.md for details. +# /// @param vocab The vocabulary that this grammar will be used with. +# /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. +# /// @param grammar_root The name of the start symbol for the grammar. # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( # const struct llama_vocab * vocab, # const char * grammar_str, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7841fc723..6bf28f011 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7841fc723e059d1fd9640e5c0ef19050fcc7c698 +Subproject commit 6bf28f0111ff9f21b3c1b1eace20c590281e7ba6 From 4c6514d365cb4f00910424c2feb8e45e68a870a6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 06:52:52 -0400 Subject: [PATCH 153/155] feat: Update llama.cpp --- CMakeLists.txt | 2 +- llama_cpp/llama_cpp.py | 33 +++++++++++++++++++++------------ vendor/llama.cpp | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64a0304a1..443d1ee53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,7 +143,7 @@ if (LLAMA_BUILD) endif() # Building llava - add_subdirectory(vendor/llama.cpp/examples/llava) + add_subdirectory(vendor/llama.cpp/tools/mtmd) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") if (WIN32) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 710bd83c8..63de3a93a 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -235,6 +235,8 @@ # LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, # LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, # LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -252,7 +254,7 @@ LLAMA_VOCAB_PRE_TYPE_DBRX = 13 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 LLAMA_VOCAB_PRE_TYPE_PORO = 15 -LLAMA_VOCAV_PRE_TYPE_CHATGLM3 = 16 +LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16 LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 LLAMA_VOCAB_PRE_TYPE_VIKING = 18 LLAMA_VOCAB_PRE_TYPE_JAIS = 19 @@ -269,6 +271,8 @@ LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 +LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33 +LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34 # // note: these values should be synchronized with ggml_rope @@ -891,17 +895,18 @@ class llama_context_params(ctypes.Structure): # // model quantization parameters # typedef struct llama_model_quantize_params { -# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() -# enum llama_ftype ftype; // quantize to this llama_ftype -# enum ggml_type output_tensor_type; // output tensor type -# enum ggml_type token_embedding_type; // token embeddings tensor type -# bool allow_requantize; // allow quantizing non-f32/f16 tensors -# bool quantize_output_tensor; // quantize output.weight -# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored -# bool pure; // quantize all tensors to the default type -# bool keep_split; // quantize to the same number of shards -# void * imatrix; // pointer to importance matrix data -# void * kv_overrides; // pointer to vector containing overrides +# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() +# enum llama_ftype ftype; // quantize to this llama_ftype +# enum ggml_type output_tensor_type; // output tensor type +# enum ggml_type token_embedding_type; // token embeddings tensor type +# bool allow_requantize; // allow quantizing non-f32/f16 tensors +# bool quantize_output_tensor; // quantize output.weight +# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored +# bool pure; // quantize all tensors to the default type +# bool keep_split; // quantize to the same number of shards +# void * imatrix; // pointer to importance matrix data +# void * kv_overrides; // pointer to vector containing overrides +# void * tensor_types; // pointer to vector containing tensor types # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -918,6 +923,7 @@ class llama_model_quantize_params(ctypes.Structure): keep_split (bool): quantize to the same number of shards imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides + tensor_types (ctypes.c_void_p): pointer to vector containing tensor types """ if TYPE_CHECKING: @@ -932,6 +938,7 @@ class llama_model_quantize_params(ctypes.Structure): keep_split: bool imatrix: ctypes.c_void_p kv_overrides: ctypes.c_void_p + tensor_types: ctypes.c_void_p _fields_ = [ ("nthread", ctypes.c_int32), @@ -945,6 +952,7 @@ class llama_model_quantize_params(ctypes.Structure): ("keep_split", ctypes.c_bool), ("imatrix", ctypes.c_void_p), ("kv_overrides", ctypes.c_void_p), + ("tensor_types", ctypes.c_void_p), ] @@ -3812,6 +3820,7 @@ def llama_sampler_init_softmax() -> llama_sampler_p: # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +# /// Setting k <= 0 makes this a noop # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) def llama_sampler_init_top_k(k: int) -> llama_sampler_p: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6bf28f011..8733e0cf6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6bf28f0111ff9f21b3c1b1eace20c590281e7ba6 +Subproject commit 8733e0cf6eefc7c7752297cc22d0836706f4222c From cb2edb9c9fef90fe680e528fadc86806c6f385e5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 06:59:55 -0400 Subject: [PATCH 154/155] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 605370e7d..affbd5db7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.9] + +- feat: Update llama.cpp to ggerganov/llama.cpp@8733e0cf6eefc7c7752297cc22d0836706f4222c + ## [0.3.8] - feat: Update llama.cpp to ggerganov/llama.cpp@7841fc723e059d1fd9640e5c0ef19050fcc7c698 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b1a8b9baa..2c9c527cd 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.8" +__version__ = "0.3.9" From b1d23df0bbd327b774083b5cf88e67ca0dd52b92 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 07:02:20 -0400 Subject: [PATCH 155/155] hotfix: Disable curl support --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 443d1ee53..b9178e856 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,9 @@ if (LLAMA_BUILD) # Enable building of the common library set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) + # Disable building curl support + set(LLAMA_CURL OFF CACHE BOOL "llama.cpp: enable curl" FORCE) + # Architecture detection and settings for Apple platforms if (APPLE) # Get the target architecture