zai-org

/

chatglm2-6b

like 2.05k

Follow

Z.ai 5.34k

Traceback (most recent call last):
  File "/data/share/code/second-life/finetune.py", line 210, in <module>
    finetune()
  File "/data/share/code/second-life/finetune.py", line 201, in finetune
    trainer.train(resume_from_checkpoint=last_checkpoint)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/transformers/trainer.py", line 1555, in train
    return inner_training_loop(
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/transformers/trainer.py", line 1837, in _inner_training_loop
    tr_loss_step = self.training_step(model, inputs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/transformers/trainer.py", line 2682, in training_step
    loss = self.compute_loss(model, inputs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/transformers/trainer.py", line 2707, in compute_loss
    outputs = model(**inputs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
    ret_val = func(*args, **kwargs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1801, in forward
    loss = self.module(*inputs, **kwargs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/peft/peft_model.py", line 529, in forward
    return self.base_model(
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/rd/.cache/huggingface/modules/transformers_modules/chatglm2-6b/modeling_chatglm.py", line 934, in forward
    transformer_outputs = self.transformer(
  File "/data/share/miniconda3/envs/llm/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/rd/.cache/huggingface/modules/transformers_modules/chatglm2-6b/modeling_chatglm.py", line 819, in forward
    full_attention_mask = self.get_masks(input_ids, past_key_values, padding_mask=attention_mask)
  File "/home/rd/.cache/huggingface/modules/transformers_modules/chatglm2-6b/modeling_chatglm.py", line 690, in get_masks
    full_attention_mask -= padding_mask.unsqueeze(-1) - 1
RuntimeError: Subtraction, the `-` operator, with a bool tensor is not supported. If you are trying to invert a mask, use the `~` or `logical_not()` operator instead.