6B-200K SFT时候 报错

[15024287710Jackson]

修改本地模型地址为上图所示，但是bash run_sft_Yi_6b.sh 之后
raceback (most recent call last):
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 415, in
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer) File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)

File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
main()
tokenizer = AutoTokenizer.from_pretrained(
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 252, in main
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
tokenizer = AutoTokenizer.from_pretrained(tokenizer = AutoTokenizer.from_pretrained(

      File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
tokenizer = AutoTokenizer.from_pretrained(tokenizer = AutoTokenizer.from_pretrained(

tokenizer = AutoTokenizer.from_pretrained( File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=False)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 58, in load_hf_tokenizer
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained

return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
Traceback (most recent call last):
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 415, in
main()
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 252, in main
tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=False)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 58, in load_hf_tokenizer
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
return cls._from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
return cls._from_pretrained(return cls._from_pretrained(return cls._from_pretrained(

return cls._from_pretrained( File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
return cls._from_pretrained(return cls._from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
return cls._from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
tokenizer = cls(*init_inputs, **init_kwargs)tokenizer = cls(*init_inputs, **init_kwargs)tokenizer = cls(*init_inputs, **init_kwargs)

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
tokenizer = cls(*init_inputs, **init_kwargs)tokenizer = cls(*init_inputs, **init_kwargs)tokenizer = cls(*init_inputs, **init_kwargs)

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor

    self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))    self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))  File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor

self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
tokenizer = cls(*init_inputs, **init_kwargs)tokenizer.Load(self.vocab_file)

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
tokenizer.Load(self.vocab_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
tokenizer.Load(self.vocab_file)
tokenizer.Load(self.vocab_file) File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load

tokenizer.Load(self.vocab_file)tokenizer.Load(self.vocab_file)tokenizer.Load(self.vocab_file) File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
tokenizer.Load(self.vocab_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
return self.LoadFromFile(model_file) return self.LoadFromFile(model_file)
return self.LoadFromFile(model_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
return self.LoadFromFile(model_file)return self.LoadFromFile(model_file)return self.LoadFromFile(model_file)return self.LoadFromFile(model_file)return self.LoadFromFile(model_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile

File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
TypeError: not a stringreturn _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)

return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)    

TypeError return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg) : not a stringreturn _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)TypeError
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)

:

not a stringTypeErrorTypeError
TypeError: TypeErrornot a string: : : TypeError
not a stringnot a stringnot a string:

not a string
[2024-04-08 09:44:31,368] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655513
[2024-04-08 09:44:31,462] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655514
[2024-04-08 09:44:31,756] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655515
[2024-04-08 09:44:31,758] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655516
[2024-04-08 09:44:31,758] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655517
[2024-04-08 09:44:31,759] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655518
[2024-04-08 09:44:31,811] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655519
[2024-04-08 09:44:31,812] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 655520

[15024287710Jackson]

[15024287710Jackson]

如果只用单张卡，

报错如下

[15024287710Jackson]

(zeroone_LLM_env) [ma-user scripts]$sh run_sft_Yi_6b.sh
[2024-04-08 11:49:46,171] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-04-08 11:49:46,957] [WARNING] [runner.py:196:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.
Detected CUDA_VISIBLE_DEVICES=0: setting --include=localhost:0
[2024-04-08 11:49:47,021] [INFO] [runner.py:555:main] cmd = /home/ma-user/anaconda3/envs/zeroone_LLM_env/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29500 --enable_each_rank_log=None main.py --data_path ../yi_example_dataset/ --model_name_or_path /home/ma-user/work/Yi-6B-200K/ --per_device_train_batch_size 1 --per_device_eval_batch_size 1 --max_seq_len 4096 --learning_rate 2e-6 --weight_decay 0. --num_train_epochs 4 --training_debug_steps 20 --gradient_accumulation_steps 1 --lr_scheduler_type cosine --num_warmup_steps 0 --seed 1234 --gradient_checkpointing --zero_stage 2 --deepspeed --offload --output_dir ./2024040801finetuned_model
[2024-04-08 11:49:48,996] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2024-04-08 11:49:49,628] [INFO] [launch.py:138:main] 0 NCCL_VERSION=2.10.3
[2024-04-08 11:49:49,628] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [0]}
[2024-04-08 11:49:49,628] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=1, node_rank=0
[2024-04-08 11:49:49,628] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0]})
[2024-04-08 11:49:49,628] [INFO] [launch.py:163:main] dist_world_size=1
[2024-04-08 11:49:49,628] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=0
[2024-04-08 11:49:51,649] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)
/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
warnings.warn(
[2024-04-08 11:49:53,393] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented
[2024-04-08 11:49:53,393] [INFO] [comm.py:616:init_distributed] cdb=None
[2024-04-08 11:49:53,393] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
lllllllllllllllll /home/ma-user/work/Yi-6B-200K/
tokenizer path exist
Traceback (most recent call last):
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 416, in
main()
File "/home/ma-user/work/Yi-main/finetune/sft/main.py", line 253, in main
tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=False)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 58, in load_hf_tokenizer
tokenizer = get_tokenizer(model_name_or_path, fast_tokenizer=fast_tokenizer)
File "/home/ma-user/work/Yi-main/finetune/utils/utils.py", line 42, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 837, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2086, in from_pretrained
return cls._from_pretrained(
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 2325, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 182, in init
self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False))
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/transformers/models/llama/tokenization_llama.py", line 209, in get_spm_processor
tokenizer.Load(self.vocab_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
return self.LoadFromFile(model_file)
File "/home/ma-user/anaconda3/envs/zeroone_LLM_env/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
TypeError: not a string
[2024-04-08 11:49:57,669] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 1063402
[2024-04-08 11:49:57,669] [ERROR] [launch.py:321:sigkill_handler] ['/home/ma-user/anaconda3/envs/zeroone_LLM_env/bin/python', '-u', 'main.py', '--local_rank=0', '--data_path', '../yi_example_dataset/', '--model_name_or_path', '/home/ma-user/work/Yi-6B-200K/', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--max_seq_len', '4096', '--learning_rate', '2e-6', '--weight_decay', '0.', '--num_train_epochs', '4', '--training_debug_steps', '20', '--gradient_accumulation_steps', '1', '--lr_scheduler_type', 'cosine', '--num_warmup_steps', '0', '--seed', '1234', '--gradient_checkpointing', '--zero_stage', '2', '--deepspeed', '--offload', '--output_dir', './2024040801finetuned_model'] exits with return code = 1