diff options
| -rwxr-xr-x | inference.py | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/inference.py b/inference.py index 67e633a..dd5af9a 100755 --- a/inference.py +++ b/inference.py @@ -58,14 +58,21 @@ def translation_prompt(source): def make_inputs(processor, messages, device): - return processor.apply_chat_template( + ret = processor.apply_chat_template( messages, - add_generation_prompt=True, - tokenize=True, + add_generation_prompt=False, + tokenize=False, return_dict=True, return_tensors="pt" ).to(device, dtype=torch.bfloat16) + print() + print(f"{ret=}") + print() + exit() + + return ret + def generate_and_parse(model, processor, |