File tree 1 file changed +3
-3
lines changed
1 file changed +3
-3
lines changed Original file line number Diff line number Diff line change 2
2
3
3
import abc
4
4
from typing import (
5
+ Any ,
5
6
List ,
6
7
Optional ,
7
- Any ,
8
8
)
9
9
10
10
import llama_cpp
@@ -62,7 +62,7 @@ def decode(self, tokens: List[int]) -> str:
62
62
return self .detokenize (tokens ).decode ("utf-8" , errors = "ignore" )
63
63
64
64
@classmethod
65
- def from_ggml_file (cls , path : str ) -> " LlamaTokenizer" :
65
+ def from_ggml_file (cls , path : str ) -> LlamaTokenizer :
66
66
return cls (llama_cpp .Llama (model_path = path , vocab_only = True ))
67
67
68
68
@@ -92,7 +92,7 @@ def detokenize(
92
92
return self .hf_tokenizer .decode (tokens ).encode ("utf-8" , errors = "ignore" )
93
93
94
94
@classmethod
95
- def from_pretrained (cls , pretrained_model_name_or_path : str ) -> " LlamaHFTokenizer" :
95
+ def from_pretrained (cls , pretrained_model_name_or_path : str ) -> LlamaHFTokenizer :
96
96
try :
97
97
from transformers import AutoTokenizer
98
98
except ImportError :
You can’t perform that action at this time.
0 commit comments