Skip to content

Commit

Permalink
Fix bad import in Baichuan code (#1547)
Browse files Browse the repository at this point in the history
  • Loading branch information
regisss authored Dec 3, 2024
1 parent bfc9233 commit 9fba7f7
Showing 1 changed file with 10 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,13 @@
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
from transformers.utils import logging
from transformers.utils.import_utils import is_sentencepiece_available


if is_sentencepiece_available():
import sentencepiece as spm


logger = logging.get_logger(__name__)
Expand Down Expand Up @@ -72,6 +76,11 @@ def __init__(
clean_up_tokenization_spaces=False,
**kwargs,
):
if not is_sentencepiece_available():
raise ModuleNotFoundError(
"Baichuan requires the Sentencepiece library to be installed. Please install it with: `pip install sentencepiece`"
)

self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
Expand Down

0 comments on commit 9fba7f7

Please sign in to comment.