From 7f96458147f9752c2b5f766e26ea0828a11d4916 Mon Sep 17 00:00:00 2001 From: cmathw Date: Mon, 22 Jan 2024 00:14:15 +0000 Subject: [PATCH] convert token id to list in .decode() --- src/transformers/models/codegen/tokenization_codegen.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/codegen/tokenization_codegen.py b/src/transformers/models/codegen/tokenization_codegen.py index b9f2b8f9b2a7..c79a6d46e4ad 100644 --- a/src/transformers/models/codegen/tokenization_codegen.py +++ b/src/transformers/models/codegen/tokenization_codegen.py @@ -23,7 +23,7 @@ import numpy as np import regex as re -from ...utils import is_tf_available, is_torch_available, logging +from ...utils import is_tf_available, is_torch_available, logging, to_py_obj if TYPE_CHECKING: @@ -352,6 +352,9 @@ def decode( Returns: `str`: The decoded sentence. """ + + token_ids = to_py_obj(token_ids) + decoded_text = super()._decode( token_ids=token_ids, skip_special_tokens=skip_special_tokens,