Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-27 21:46:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

gguf-py : ByteLevel processing defaults bos/eos to False (#29422)

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2026-09-25 13:59:47 +02:00
committed by GitHub
parent d81aef1994
commit b248f4a3c1
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/vocab.py
+7 -3
View File
@@ -234,7 +234,11 @@ class SpecialVocab:
tokenizer_config['eos_token'] = special_eos = special_sep
if post_processor := tokenizer.get('post_processor'):
for processor in post_processor.get('processors', [post_processor]):
if processor.get('type') == 'RobertaProcessing':
processor_type = processor.get('type')
if processor_type == 'ByteLevel':
self.add_special_token.setdefault('bos', False)
self.add_special_token.setdefault('eos', False)
elif processor_type == 'RobertaProcessing':
self.add_special_token['bos'] = True
self.add_special_token['eos'] = True
self.add_special_token['sep'] = True
@@ -247,7 +251,7 @@ class SpecialVocab:
continue
# Crude parsing of TemplateProcessing to determine if BOS/SEP/EOS should be added
# Only works with simple templates, **will** get it wrong on unusual sequences
if processor.get('type') == 'TemplateProcessing':
elif processor_type == 'TemplateProcessing':
tmpl_single = processor.get('single', [])
tmpl_pair = processor.get('pair', [])
special_first = None
@@ -339,7 +343,7 @@ class SpecialVocab:
if typ not in self.add_special_token:
self.add_special_token[typ] = add_entry
elif self.add_special_token[typ] != add_entry:
logger.warning(f'Mismatch between tokenizer_config add_{typ}_token({add_entry}) and TemplateProcessing<{typ}>({self.add_special_token[typ]}) - opting for the latter')
logger.warning(f'Mismatch between tokenizer_config add_{typ}_token({add_entry}) and tokenizer post_processor<{typ}>({self.add_special_token[typ]}) - opting for the latter')
entry = tokenizer_config.get(f'{typ}_token')
if isinstance(entry, str):
tc_content = entry