ckpt

swngui · swngui · commit 6519423c08ba · 2024-04-09T01:56:49.000+08:00
diff --git a/.gitignore b/.gitignore
@@ -181,3 +181,4 @@ compile_commands.json
 /bin
 cmake_install.cmake
 CMakeCache.txt
+/.fwd_cache
diff --git a/llama_cpp/managers/__init__.py b/llama_cpp/managers/__init__.py
@@ -1,2 +1,2 @@
 from .state import LlamaPersistantState, STATE_PROMPTS
-from .cache import _LlamaCacheManager, CACHE_TYPES
+from .cache import LlamaCacheManager, CACHE_TYPES
diff --git a/llama_cpp/managers/cache.py b/llama_cpp/managers/cache.py
@@ -22,7 +22,7 @@ class CACHE_TYPES(BaseModel):
     cache_type: Literal["disk", "ram"]
 
 
-class _LlamaCacheManager(Llama):
+class LlamaCacheManager(Llama):
     """A class for an LLM to always use a specific state with a prompt.
     This should be inherited by a strategy class and not used directly."""
 
diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py
@@ -9,7 +9,7 @@
 import llama_cpp.llama_tokenizer as llama_tokenizer
 
 from llama_cpp.server.settings import ModelSettings
-
+from llama_cpp.managers.cache import LlamaCacheManager
 
 class LlamaProxy:
     def __init__(self, models: List[ModelSettings]) -> None:
@@ -132,7 +132,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 filename=settings.model,
             )
         else:
-            create_fn = llama_cpp.Llama
+            create_fn = LlamaCacheManager
             kwargs["model_path"] = settings.model
 
         _model = create_fn(
diff --git a/llama_cpp/strategies/chat_history.py b/llama_cpp/strategies/chat_history.py
@@ -1,7 +1,7 @@
-from ..managers.cache import _LlamaCacheManager, CACHE_TYPES
+from ..managers.cache import CACHE_TYPES, LlamaCacheManager
 
 
-class ChatHistoryStrategy(_LlamaCacheManager):
+class ChatHistoryStrategy(LlamaCacheManager):
     """A class for an LLM to always use a specific state with a prompt."""
 
     def __init__(

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`from .state import LlamaPersistantState, STATE_PROMPTS`
`2`		`-from .cache import _LlamaCacheManager, CACHE_TYPES`
	`2`	`+from .cache import LlamaCacheManager, CACHE_TYPES`