# Molformer model training error

**URL:** <https://discuss.huggingface.co/t/molformer-model-training-error/147291>\
**Category:** 🤗Transformers\
**Created:** [March 25, 2025, 6:45am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291 "2025-03-25T06:45:45Z")\
**Posts on this page:** 7\
**Page:** 1

<div class="post-metadata">

**Author:** ![nypabhishek](https://avatars.discourse-cdn.com/v4/letter/n/e9a140/32.png) [@nypabhishek](https://discuss.huggingface.co/u/nypabhishek)\
**Post date:** [March 25, 2025, 6:45am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/1 "2025-03-25T06:45:46Z")

</div>

below is the training script that i have been using to train the model  
ibm-research/MoLFormer-XL-both-10pct  
on my custom dataset of 6 rows of training and 6 rows of validation

now the error that is arising is due to saving of the tensors of the model after the training that i think of

error

```auto
You are trying to save a non contiguous tensor: `base_model.encoder.layer.0.attention.self.feature_map.weight` which is not allowed. It either means you are trying to save tensors which are reference of each other in which case it's recommended to save only the full tensors, and reslice at load time, or simply call `.contiguous()` on your tensor to pack it before saving.

```

to takle this i added the for loop of contiguous tensors but nothing to avail and my guess is that the argument of save strategy in training arguments is the culprit

i did changed it to no but then nothing get stored in the results as it should be.  
please any help would be helpful

```auto

import torch
from transformers import AutoTokenizer, AutoModel, Trainer, TrainingArguments
from datasets import Dataset
import torch.nn as nn

# Load model and tokenizer
model_name = "ibm-research/MoLFormer-XL-both-10pct"
tokenizer = AutoTokenizer.from_pretrained(model_name,trust_remote_code=True)
base_model = AutoModel.from_pretrained(model_name,trust_remote_code=True)
for param in base_model.parameters():
        if not param.is_contiguous():
            param.data = param.data.contiguous()

descriptor_list = [
    "alcoholic", "aldehydic", "alliaceous", "almond", "amber", "animal", "anisic", "apple", "apricot", "aromatic", "balsamic", "banana", "beefy", "bergamot", "berry", "bitter", "black currant", "brandy", "burnt", "buttery", "cabbage", "camphoreous", "caramellic", "cedar", "celery", "chamomile", "cheesy", "cherry", "chocolate", "cinnamon", "citrus", "clean", "clove", "cocoa", "coconut", "coffee", "cognac", "cooked", "cooling", "cortex", "coumarinic", "creamy", "cucumber", "dairy", "dry", "earthy", "ethereal", "fatty", "fermented", "fishy", "floral", "fresh", "fruit skin", "fruity", "garlic", "gassy", "geranium", "grape", "grapefruit", "grassy", "green", "hawthorn", "hay", "hazelnut", "herbal", "honey", "hyacinth", "jasmin", "juicy", "ketonic", "lactonic", "lavender", "leafy", "leathery", "lemon", "lily", "malty", "meaty", "medicinal", "melon", "metallic", "milky", "mint", "muguet", "mushroom", "musk", "musty", "natural", "nutty", "odorless", "oily", "onion", "orange", "orangeflower", "orris", "ozone", "peach", "pear", "phenolic", "pine", "pineapple", "plum", "popcorn", "potato", "powdery", "pungent", "radish", "raspberry", "ripe", "roasted", "rose", "rummy", "sandalwood", "savory", "sharp", "smoky", "soapy", "solvent", "sour", "spicy", "strawberry", "sulfurous", "sweaty", "sweet", "tea", "terpenic", "tobacco", "tomato", "tropical", "vanilla", "vegetable", "vetiver", "violet", "warm", "waxy", "weedy", "winey", "woody"
]
label_map = {desc: i for i, desc in enumerate(descriptor_list)}

# Training dataset
train_data = [
    ("CC(O)CN", "fishy"),
    ("CCC(=O)C(=O)O", "fatty;lactonic;sweet;caramellic;creamy"),
    ("O=C(O)CCc1ccccc1", "rose;floral;fatty;sweet;musk;cinnamon;balsamic"),
    ("OCc1ccc(O)cc1", "medicinal;phenolic;fruity;nutty;bitter;sweet;almond;coconut"),
    ("O=Cc1ccc(O)cc1", "phenolic;woody;nutty;vanilla;hay;metallic;sweet;almond;honey;balsamic")
]

# Validation dataset
val_data = [
    ("O=C(O)c1ccc(O)cc1", "nutty;phenolic"),
    ("CC(=O)O", "pungent;sharp;sour"),
    ("CC=O", "fresh;fruity;nutty;alcoholic;pungent;aldehydic;musty;ethereal"),
    ("CC(=O)C(C)O", "fatty;sweet;milky;dairy;buttery;creamy"),
    ("CC(C)=O", "pear;apple;ethereal;solvent"),
    ("O=C(O)CCCCC(=O)O", "sour")
]

def encode_data(data):
    inputs, masks, labels = [], [], []
    for smiles, descriptor_str in data:
        tokenized = tokenizer(smiles, truncation=True, padding="max_length", max_length=128, return_tensors="pt")
        label_tensor = torch.zeros(len(label_map))
        for d in descriptor_str.split(";"):
            if d in label_map:
                label_tensor[label_map[d]] = 1
        inputs.append(tokenized["input_ids"].squeeze().tolist())
        masks.append(tokenized["attention_mask"].squeeze().tolist())
        labels.append(label_tensor.tolist())
    return Dataset.from_dict({"input_ids": inputs, "attention_mask": masks, "labels": labels})

# Create datasets
dataset = encode_data(train_data)
val_dataset = encode_data(val_data)

# Define Multi-Label Model
class MultiLabelModel(nn.Module):
    def __init__ (self, base_model, num_labels):
        super(). __init__ ()
        self.base_model = base_model
        self.classifier = nn.Linear(base_model.config.hidden_size, num_labels)

    def forward(self, input_ids, attention_mask, labels=None):
        outputs = self.base_model(input_ids=input_ids, attention_mask=attention_mask)
        logits = self.classifier(outputs.last_hidden_state[:, 0, :]) # CLS token output
        logits = logits.contiguous()
        if labels is not None:
            loss_fn = nn.BCEWithLogitsLoss()
            loss = loss_fn(logits, labels)
            return {"loss": loss, "logits": logits}
        return {"logits": logits}

num_labels = len(label_map)
model = MultiLabelModel(base_model, num_labels)

# Training arguments
training_args = TrainingArguments(
    output_dir="./results",
    per_device_train_batch_size=1,
    num_train_epochs=10,
    logging_dir="./logs",
    logging_steps=1,
    #################save_strategy = "epoch"
)

# Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    eval_dataset=val_dataset
)
trainer.train()

```

---

<div class="post-metadata">

**Author:** ![nypabhishek](https://avatars.discourse-cdn.com/v4/letter/n/e9a140/32.png) [@nypabhishek](https://discuss.huggingface.co/u/nypabhishek)\
**Post date:** [March 25, 2025, 6:51am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/2 "2025-03-25T06:51:49Z")

</div>

@John6666 can you please help in this 🙏

---

<div class="post-metadata">

**Author:** ![John6666](https://sea2.discourse-cdn.com/hellohellohello/user_avatar/discuss.huggingface.co/john6666/32/27664_2.png) [@John6666](https://discuss.huggingface.co/u/John6666)\
**Post date:** [March 25, 2025, 7:46am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/3 "2025-03-25T07:46:55Z")

</div>

It’s probably possible to get around this, but it would be even better if you could find out why it’s no longer contiguous and fix it.

> **[Trainer](https://huggingface.co/docs/transformers/en/main_classes/trainer#transformers.TrainingArguments.save_safetensors)**
>
> We’re on a journey to advance and democratize artificial intelligence through open source and open science.

```py
training_args = TrainingArguments(
    output_dir="./results",
    per_device_train_batch_size=1,
    num_train_epochs=10,
    logging_dir="./logs",
    logging_steps=1,
    save_safetensors=False, # added
    #################save_strategy = "epoch"
)

```

> [@Using Trainer to save a Bartforsequenceclassification model](https://discuss.huggingface.co/t/using-trainer-to-save-a-bartforsequenceclassification-model/81606):
>
> trainer.train() trainer.save\_model() It gives me the error: modeling\_utils.py", line 2546, in save\_pretrained raise RuntimeError( RuntimeError: The weights trying to be saved contained shared tensors [{‘model.shared.weight’, ‘model.encoder.embed\_tokens.weight’, ‘model.decoder.embed\_tokens.weight’}] that are mismatching the transformers base configuration. Try saving using safe\_serialization=False or remove this tensor sharing. Could anyone please tell me why is it happening? and how to so…

> <https://github.com/huggingface/transformers/issues/28293>
>
> \### System Info
> 
> Transformers version: 4.36.2
> pytorch version: 2.1.1
> Python… version: 3.10.13
> 
> \### Who can help?
> 
> \_No response\_
> 
> \### Information
> 
> \- \[\] The official example scripts
> \- \[X\] My own modified scripts
> 
> \### Tasks
> 
> \- \[\] An officially supported task in the \`examples\` folder (such as GLUE/SQuAD, ...)
> \- \[X\] My own task or dataset (give details below)
> 
> \### Reproduction
> 
> 
> Fine-tuning mt5 model on a task using transformers trainer, and try to save the model, then the following error occurs. 
>  
> \`\`\`
> File "/home/xxx/xxx/xxx/run.py", line 17, in main
> experiment.run()
> File "/home/xxx/xxx/xxx/experiments.py", line 162, in run
> self.train()
> File "/home/xxx/xxx/xxx/experiments.py", line 207, in train
> trainer.save\_model() # Saves the tokenizer too for easy upload
> File "/home/xxx/.local/lib/python3.10/site-packages/transformers/trainer.py", line 2849, in save\_model
> self.\_save(output\_dir)
> File "/home/xxx/.local/lib/python3.10/site-packages/transformers/trainer.py", line 2909, in \_save
> self.model.save\_pretrained(
> File "/home/xxx/.local/lib/python3.10/site-packages/transformers/modeling\_utils.py", line 2376, in save\_pretrained
> safe\_save\_file(shard, os.path.join(save\_directory, shard\_file), metadata={"format": "pt"})
> File "/home/xxx/.local/lib/python3.10/site-packages/safetensors/torch.py", line 281, in save\_file
> serialize\_file(\_flatten(tensors), filename, metadata=metadata)
> File "/home/xxx/.local/lib/python3.10/site-packages/safetensors/torch.py", line 475, in \_flatten
> return {
> File "/home/xxx/.local/lib/python3.10/site-packages/safetensors/torch.py", line 479, in \<dictcomp\>
> "data": \_tobytes(v, k),
> File "/home/xxx/.local/lib/python3.10/site-packages/safetensors/torch.py", line 396, in \_tobytes
> raise ValueError(
> ValueError: You are trying to save a non contiguous tensor: \`encoder\_decoder.encoder.block.0.layer.0.SelfAttention.q.weight\` which is not allowed. It either means you are trying to save tensors which are reference of each other in which case it's recommended to save only the full tensors, and reslice at load time, or simply call \`.contiguous()\` on your tensor to pack it before saving.
> \`\`\`
> 
> \### Expected behavior
> 
> Fine-tune mt5 model, and try to save the fine-tuned model, it renders the above error, and modifying \`transformers/modeling\_utils.py\` file with \`state\_dict= {k:v.contiguous() for k,v in state\_dict.items()}\` solves the problem.

---

<div class="post-metadata">

**Author:** ![John6666](https://sea2.discourse-cdn.com/hellohellohello/user_avatar/discuss.huggingface.co/john6666/32/27664_2.png) [@John6666](https://discuss.huggingface.co/u/John6666)\
**Post date:** [March 25, 2025, 7:57am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/4 "2025-03-25T07:57:05Z")

</div>

Hmm…

* * *

To address the issue of non-contiguous tensors in your PyTorch model, follow these structured steps to ensure all tensors are contiguous before saving and during operations:

### Step 1: Identify Sources of Non-Contiguous Tensors

Check your model’s attention layer or any operations that might produce non-contiguous tensors. For example, transposing, slicing, or reshaping operations can make tensors non-contiguous.

### Step 2: Modify the Model’s Forward Pass

After any operation that could result in a non-contiguous tensor, explicitly make the tensor contiguous. For instance, after the base model’s output:

```python
outputs = self.base_model(input_ids=input_ids, attention_mask=attention_mask)
last_hidden_state = outputs.last_hidden_state
# Ensure contiguous
last_hidden_state = last_hidden_state.contiguous()
logits = self.classifier(last_hidden_state[:, 0, :])
logits = logits.contiguous()

```

### Step 3: Override the `save_pretrained` Method

Ensure that all tensors in the model’s state dictionary are contiguous before saving. Modify your `MultiLabelModel` class as follows:

```python
class MultiLabelModel(nn.Module):
    def __init__ (self, base_model, num_labels):
        super(). __init__ ()
        self.base_model = base_model
        self.classifier = nn.Linear(base_model.config.hidden_size, num_labels)

    def forward(self, input_ids, attention_mask, labels=None):
        outputs = self.base_model(input_ids=input_ids, attention_mask=attention_mask)
        last_hidden_state = outputs.last_hidden_state
        last_hidden_state = last_hidden_state.contiguous() # Ensure contiguous
        logits = self.classifier(last_hidden_state[:, 0, :])
        logits = logits.contiguous() # Ensure contiguous

        if labels is not None:
            loss_fn = nn.BCEWithLogitsLoss()
            loss = loss_fn(logits, labels)
            return {"loss": loss, "logits": logits}
        return {"logits": logits}

    def save_pretrained(self, save_directory):
        # Ensure all tensors are contiguous before saving
        state_dict = self.state_dict()
        contiguous_state_dict = {}
        for key, tensor in state_dict.items():
            if not tensor.is_contiguous():
                contiguous_state_dict[key] = tensor.contiguous()
            else:
                contiguous_state_dict[key] = tensor
        super().save_pretrained(save_directory, state_dict=contiguous_state_dict)

```

### Step 4: Configure Training Arguments for Contiguity

Ensure that your training arguments are set to save weights or the entire model as needed. You can adjust the `save_strategy` to “epoch” to save models at the end of each epoch.

```python
training_args = TrainingArguments(
    output_dir="./results",
    per_device_train_batch_size=1,
    num_train_epochs=10,
    logging_dir="./logs",
    logging_steps=1,
    save_strategy="epoch",
    save_steps=100,
    save_total_limit=2,
)

```

### Step 5: Use `save_weights_only` Option (Optional)

If saving the entire model causes issues, consider saving only the weights:

```python
model.save_weightsOnly(save_directory="./results")

```

Ensure that any custom save methods check and make tensors contiguous before saving.

### Summary

By ensuring that all tensors are contiguous during the forward pass and before saving, you can prevent the non-contiguous tensor error. Modifying the `save_pretrained` method to enforce contiguity and adjusting training arguments to save models correctly are key steps in resolving this issue.

---

<div class="post-metadata">

**Author:** ![nypabhishek](https://avatars.discourse-cdn.com/v4/letter/n/e9a140/32.png) [@nypabhishek](https://discuss.huggingface.co/u/nypabhishek)\
**Post date:** [March 25, 2025, 8:21am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/5 "2025-03-25T08:21:27Z")

</div>

thanks @John6666  
using the `save_safetensors=False` argument helped me overcome the error  
but making the tensors contiguous explicitly still resulted in the same error where could the problem lie??

---

<div class="post-metadata">

**Author:** ![John6666](https://sea2.discourse-cdn.com/hellohellohello/user_avatar/discuss.huggingface.co/john6666/32/27664_2.png) [@John6666](https://discuss.huggingface.co/u/John6666)\
**Post date:** [March 25, 2025, 9:48am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/6 "2025-03-25T09:48:35Z")

</div>

It seems that this may be caused by the Accelerate library, but I don’t know if this has been resolved yet…

> <https://stackoverflow.com/questions/48915810/what-does-contiguous-do-in-pytorch>

> <https://github.com/deepspeedai/DeepSpeed/issues/2736>
>
> I am just trying to fine-tune "EleutherAI/gpt-neo-1.3B" for casualLM on google c…olab. Without anything, it gives out of memory error. I was checking what can I do and I found deepspeed. I added deepspeed='ds\_config.json', to my training arguments in jupyter notebook and used configuration from the official page "ds\_config\_zero2.json". After that, I start to get this error. I am trying to do it in the notebook, not as a command.
> 
> \*\*To Reproduce\*\*
> try fine-tuning gpt-neo
> 
> This is the full error 
> \`\`\`
> The following columns in the training set don't have a corresponding argument in \`GPTNeoForCausalLM.forward\` and have been ignored: text. If text are not expected by \`GPTNeoForCausalLM.forward\`, you can safely ignore this message.
> \[2023-01-23 12:41:08,453\] \[INFO\] \[logging.py:68:log\_dist\] \[Rank 0\] DeepSpeed info: version=0.8.0, git-hash=unknown, git-branch=unknown
> \---------------------------------------------------------------------------
> RuntimeError Traceback (most recent call last)
> \[\<ipython-input-21-3435b262f1ae\>\](https://localhost:8080/#) in \<module\>
> \----\> 1 trainer.train()
> 
> 10 frames
> \[/usr/local/lib/python3.8/dist-packages/transformers/trainer.py\](https://localhost:8080/#) in train(self, resume\_from\_checkpoint, trial, ignore\_keys\_for\_eval, \*\*kwargs)
> 1525 self.\_inner\_training\_loop, self.\_train\_batch\_size, args.auto\_find\_batch\_size
> 1526 )
> \-\> 1527 return inner\_training\_loop(
> 1528 args=args,
> 1529 resume\_from\_checkpoint=resume\_from\_checkpoint,
> 
> \[/usr/local/lib/python3.8/dist-packages/transformers/trainer.py\](https://localhost:8080/#) in \_inner\_training\_loop(self, batch\_size, args, resume\_from\_checkpoint, trial, ignore\_keys\_for\_eval)
> 1594 )
> 1595 if args.deepspeed:
> \-\> 1596 deepspeed\_engine, optimizer, lr\_scheduler = deepspeed\_init(
> 1597 self, num\_training\_steps=max\_steps, resume\_from\_checkpoint=resume\_from\_checkpoint
> 1598 )
> 
> \[/usr/local/lib/python3.8/dist-packages/transformers/deepspeed.py\](https://localhost:8080/#) in deepspeed\_init(trainer, num\_training\_steps, resume\_from\_checkpoint, inference)
> 342 )
> 343 
> \--\> 344 deepspeed\_engine, optimizer, \_, lr\_scheduler = deepspeed.initialize(\*\*kwargs)
> 345 
> 346 if resume\_from\_checkpoint is not None:
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/\_\_init\_\_.py\](https://localhost:8080/#) in initialize(args, model, optimizer, model\_parameters, training\_data, lr\_scheduler, mpu, dist\_init\_required, collate\_fn, config, config\_params)
> 123 
> 124 if not isinstance(model, PipelineModule):
> \--\> 125 engine = DeepSpeedEngine(args=args,
> 126 model=model,
> 127 optimizer=optimizer,
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/runtime/engine.py\](https://localhost:8080/#) in \_\_init\_\_(self, args, model, optimizer, model\_parameters, training\_data, lr\_scheduler, mpu, dist\_init\_required, collate\_fn, config, config\_params, dont\_change\_device)
> 299 
> 300 # Configure distributed model
> \--\> 301 self.\_configure\_distributed\_model(model)
> 302 
> 303 self.\_get\_model\_parameters()
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/runtime/engine.py\](https://localhost:8080/#) in \_configure\_distributed\_model(self, model)
> 1185 
> 1186 if not self.amp\_enabled():
> \-\> 1187 self.\_broadcast\_model()
> 1188 
> 1189 # check if parameters are duplicated in optimizer param\_groups
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/runtime/engine.py\](https://localhost:8080/#) in \_broadcast\_model(self)
> 1100 else:
> 1101 if torch.is\_tensor(p) and is\_replicated(p):
> \-\> 1102 dist.broadcast(p,
> 1103 groups.\_get\_broadcast\_src\_rank(),
> 1104 group=self.data\_parallel\_group)
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/comm/comm.py\](https://localhost:8080/#) in log\_wrapper(\*args, \*\*kwargs)
> 125 # Return the op, then stop the op's timer
> 126 try:
> \--\> 127 return func(\*args, \*\*kwargs)
> 128 finally:
> 129 if comms\_logger.enabled:
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/comm/comm.py\](https://localhost:8080/#) in broadcast(tensor, src, group, async\_op, prof, log\_name, debug)
> 230 debug=get\_caller\_func()):
> 231 global cdb
> \--\> 232 return cdb.broadcast(tensor=tensor, src=src, group=group, async\_op=async\_op)
> 233 
> 234 
> 
> \[/usr/local/lib/python3.8/dist-packages/deepspeed/comm/torch.py\](https://localhost:8080/#) in broadcast(self, tensor, src, group, async\_op)
> 68 
> 69 def broadcast(self, tensor, src, group=None, async\_op=False):
> \---\> 70 return torch.distributed.broadcast(tensor=tensor,
> 71 src=src,
> 72 group=group,
> 
> \[/usr/local/lib/python3.8/dist-packages/torch/distributed/distributed\_c10d.py\](https://localhost:8080/#) in broadcast(tensor, src, group, async\_op)
> 1402 group\_src\_rank = get\_group\_rank(group, src)
> 1403 opts.rootRank = group\_src\_rank
> \-\> 1404 work = group.broadcast(\[tensor\], opts)
> 1405 if async\_op:
> 1406 return work
> 
> RuntimeError: Tensors must be contiguous
> \`\`\`
> 
> \*\*ds\_report output\*\*
> \`\`\`
> DeepSpeed C++/CUDA extension op report
> \--------------------------------------------------
> NOTE: Ops not installed will be just-in-time (JIT) compiled at
> runtime if needed. Op compatibility means that your system
> meet the required dependencies to JIT install the op.
> \--------------------------------------------------
> JIT compiled ops requires ninja
> ninja .................. \[OKAY\]
> \--------------------------------------------------
> op name ................ installed .. compatible
> \--------------------------------------------------
> \[WARNING\] async\_io requires the dev libaio .so object and headers but these were not found.
> \[WARNING\] async\_io: please install the libaio-dev package with apt
> \[WARNING\] If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.
> async\_io ............... \[NO\] ....... \[NO\]
> cpu\_adagrad ............ \[NO\] ....... \[OKAY\]
> cpu\_adam ............... \[NO\] ....... \[OKAY\]
> fused\_adam ............. \[NO\] ....... \[OKAY\]
> fused\_lamb ............. \[NO\] ....... \[OKAY\]
> quantizer .............. \[NO\] ....... \[OKAY\]
> random\_ltd ............. \[NO\] ....... \[OKAY\]
> \[WARNING\] please install triton==1.0.0 if you want to use sparse attention
> sparse\_attn ............ \[NO\] ....... \[NO\]
> spatial\_inference ...... \[NO\] ....... \[OKAY\]
> transformer ............ \[NO\] ....... \[OKAY\]
> stochastic\_transformer . \[NO\] ....... \[OKAY\]
> transformer\_inference .. \[NO\] ....... \[OKAY\]
> utils .................. \[NO\] ....... \[OKAY\]
> \--------------------------------------------------
> DeepSpeed general environment info:
> torch install path ............... \['/usr/local/lib/python3.8/dist-packages/torch'\]
> torch version .................... 1.13.1+cu116
> deepspeed install path ........... \['/usr/local/lib/python3.8/dist-packages/deepspeed'\]
> deepspeed info ................... 0.8.0, unknown, unknown
> torch cuda version ............... 11.6
> torch hip version ................ None
> nvcc version ..................... 11.2
> deepspeed wheel compiled w. ...... torch 1.13, cuda 11.6
> \`\`\`
> 
> 
> \*\*System info (please complete the following information):\*\*
> google colab
> 
> \*\*Launcher context\*\*
> Are you launching your experiment with the \`deepspeed\` launcher, MPI, or something else?
> 
> \*\*Docker context\*\*
> Are you using a specific docker image that you can share?
> 
> \*\*Additional context\*\*
> Add any other context about the problem here.

> <https://github.com/huggingface/accelerate/issues/1764>
>
> \### System Info
> 
> \`\`\`Shell
> Libraries:
> pytorch 2.0.1
> pytorch-c…uda 11.8
> numpy 1.24.3
> accelerate 0.21.0
> \`\`\`
> 
> 
> \### Information
> 
> \- \[\] The official example scripts
> \- \[\] My own modified scripts
> 
> \### Tasks
> 
> \- \[\] One of the scripts in the examples/ folder of Accelerate or an officially supported \`no\_trainer\` script in the \`examples\` folder of the \`transformers\` repo (such as \`run\_no\_trainer\_glue.py\`)
> \- \[X\] My own task or dataset (give details below)
> 
> \### Reproduction
> 
> When using the following accelerate config my code crashes as shown in the screenshot below:
> \`\`\`
> 1 compute\_environment: LOCAL\_MACHINE
> 2 distributed\_type: MULTI\_GPU
> 3 downcast\_bf16: 'no'
> 4 gpu\_ids: all
> 5 machine\_rank: 0
> 6 main\_training\_function: main
> 7 mixed\_precision: 'no'
> 8 num\_machines: 1
> 9 num\_processes: 1
> 10 rdzv\_backend: static
> 11 same\_network: true
> 12 tpu\_env: \[\]
> 13 tpu\_use\_cluster: false
> 14 tpu\_use\_sudo: false
> 15 use\_cpu: false
> 16 main\_process\_port: 8080
> \`\`\`
> 
> \<img width="817" alt="image" src="https://github.com/huggingface/accelerate/assets/65319353/1518ffc5-a5d4-4296-a4ef-b6a3427f8ebd"\>
> 
> \### Expected behavior
> 
> Whereas when using the following accelerate config, the code runs perfectly:
> 
> \`\`\`
> 1 compute\_environment: LOCAL\_MACHINE
> 2 distributed\_type: MULTI\_GPU
> 3 downcast\_bf16: 'no'
> 4 gpu\_ids: all
> 5 machine\_rank: 0
> 6 main\_training\_function: main
> 7 mixed\_precision: fp16
> 8 num\_machines: 1
> 9 num\_processes: 1
> 10 rdzv\_backend: static
> 11 same\_network: true
> 12 tpu\_env: \[\]
> 13 tpu\_use\_cluster: false
> 14 tpu\_use\_sudo: false
> 15 use\_cpu: false
> 16 main\_process\_port: 8080
> \`\`\`
> 
> Doing some online search, I came across this issue https://github.com/huggingface/transformers/issues/20942 from the transformers repo. 
> 
> Something funny is happening with mixed precision setup. I hope you could help me sort this bug out!
> 
> Thank you

> <https://stackoverflow.com/questions/78019134/how-to-properly-save-the-finetuned-transformer-model-in-safetensors-without-losi>

> <https://github.com/huggingface/transformers/issues/27293>
>
> \### System Info
> 
> \- \`transformers\` version: 4.36.0.dev0
> \- Platform: Linux-4.19.0…-25-cloud-amd64-x86\_64-with-glibc2.28
> \- Python version: 3.9.17
> \- Huggingface\_hub version: 0.16.4
> \- Safetensors version: 0.3.2
> \- Accelerate version: 0.24.1
> \- Accelerate config: not found
> \- PyTorch version (GPU?): 2.0.1+cu118 (True)
> \- Tensorflow version (GPU?): not installed (NA)
> \- Flax version (CPU?/GPU?/TPU?): not installed (NA)
> \- Jax version: not installed
> \- JaxLib version: not installed
> \- Using GPU in script?: 8\*A100
> \- Using distributed or parallel set-up in script?: accelerate + deepspeed zero3
> 
> \### Who can help?
> 
> \_No response\_
> 
> \### Information
> 
> \- \[\] The official example scripts
> \- \[\] My own modified scripts
> 
> \### Tasks
> 
> \- \[\] An officially supported task in the \`examples\` folder (such as GLUE/SQuAD, ...)
> \- \[\] My own task or dataset (give details below)
> 
> \### Reproduction
> 
> I am finetuning Fuyu-8B and found the code for calling \`model.save\_pretrained\` method would run into error after upgrading to \`4.36.0\`.
> 
> The error shows:
> \`\`\`
> Removed shared tensor {'language\_model.model.layers.12.self\_attn.dense.weight', 'language\_model.model.layers.22.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.24.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.15.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.22.input\_layernorm.weight', 'language\_model.model.layers.25.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.8.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.33.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.6.post\_attention\_layernorm.weight', 'language\_model.model.layers.30.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.5.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.10.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.5.post\_attention\_layernorm.weight', 'language\_model.model.layers.15.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.2.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.4.input\_layernorm.bias', 'language\_model.model.layers.25.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.29.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.13.self\_attn.query\_key\_value.bias', 'language\_model.lm\_head.weight', 'language\_model.model.layers.6.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.13.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.14.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.31.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.32.input\_layernorm.weight', 'language\_model.model.layers.19.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.24.self\_attn.dense.bias', 'language\_model.model.layers.5.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.7.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.10.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.18.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.29.post\_attention\_layernorm.bias', 'language\_model.model.layers.11.self\_attn.dense.weight', 'language\_model.model.layers.28.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.14.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.15.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.35.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.17.post\_attention\_layernorm.bias', 'language\_model.model.layers.23.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.15.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.final\_layernorm.weight', 'language\_model.model.layers.6.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.29.input\_layernorm.weight', 'language\_model.model.layers.13.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.6.self\_attn.dense.weight', 'language\_model.model.layers.22.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.35.post\_attention\_layernorm.bias', 'language\_model.model.layers.23.self\_attn.dense.bias', 'language\_model.model.layers.16.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.32.self\_attn.dense.weight', 'language\_model.model.layers.25.self\_attn.dense.bias', 'language\_model.model.layers.9.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.25.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.3.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.21.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.32.post\_attention\_layernorm.bias', 'language\_model.model.layers.33.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.2.post\_attention\_layernorm.bias', 'language\_model.model.layers.20.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.4.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.29.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.32.self\_attn.dense.bias', 'language\_model.model.layers.8.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.34.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.35.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.4.post\_attention\_layernorm.bias', 'language\_model.model.layers.28.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.8.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.32.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.28.self\_attn.dense.weight', 'language\_model.model.layers.31.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.0.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.11.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.29.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.19.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.12.post\_attention\_layernorm.weight', 'language\_model.model.layers.7.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.13.input\_layernorm.weight', 'language\_model.model.layers.31.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.0.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.34.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.1.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.35.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.29.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.34.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.30.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.0.input\_layernorm.bias', 'language\_model.model.layers.18.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.1.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.26.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.8.post\_attention\_layernorm.weight', 'language\_model.model.layers.18.self\_attn.dense.bias', 'language\_model.model.layers.30.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.7.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.31.self\_attn.dense.weight', 'language\_model.model.layers.9.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.12.input\_layernorm.bias', 'language\_model.model.layers.14.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.28.self\_attn.dense.bias', 'language\_model.model.layers.6.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.30.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.11.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.33.self\_attn.dense.bias', 'language\_model.model.layers.14.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.14.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.12.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.10.self\_attn.dense.weight', 'language\_model.model.layers.5.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.33.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.17.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.19.self\_attn.dense.bias', 'language\_model.model.layers.4.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.19.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.8.input\_layernorm.bias', 'language\_model.model.layers.6.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.31.self\_attn.dense.bias', 'language\_model.model.layers.25.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.34.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.7.input\_layernorm.bias', 'language\_model.model.layers.2.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.29.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.16.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.35.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.35.post\_attention\_layernorm.weight', 'language\_model.model.layers.1.self\_attn.dense.weight', 'language\_model.model.layers.4.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.15.input\_layernorm.bias', 'language\_model.model.layers.4.post\_attention\_layernorm.weight', 'language\_model.model.layers.14.input\_layernorm.weight', 'language\_model.model.layers.22.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.11.input\_layernorm.weight', 'language\_model.model.layers.27.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.18.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.25.mlp.dense\_h\_to\_4h.bias', 'language\_model.model.layers.32.input\_layernorm.bias', 'language\_model.model.layers.10.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.14.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.8.post\_attention\_layernorm.bias', 'language\_model.model.layers.27.self\_attn.dense.bias', 'language\_model.model.layers.21.self\_attn.k\_layernorm.weight', 'language\_model.model.layers.27.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.30.self\_attn.dense.weight', 'language\_model.model.layers.23.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.18.post\_attention\_layernorm.weight', 'language\_model.model.layers.22.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.13.self\_attn.dense.bias', 'language\_model.model.layers.14.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.10.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.34.input\_layernorm.bias', 'language\_model.model.layers.3.post\_attention\_layernorm.bias', 'language\_model.model.layers.5.input\_layernorm.weight', 'language\_model.model.layers.8.self\_attn.query\_key\_value.weight', 'language\_model.model.layers.27.post\_attention\_layernorm.bias', 'language\_model.model.layers.28.mlp.dense\_h\_to\_4h.weight', 'language\_model.model.layers.28.self\_attn.q\_layernorm.weight', 'language\_model.model.layers.5.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.19.self\_attn.dense.weight', 'language\_model.model.layers.21.input\_layernorm.weight', 'language\_model.model.layers.14.post\_attention\_layernorm.bias', 'language\_model.model.layers.35.self\_attn.query\_key\_value.bias', 'language\_model.model.layers.10.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.17.self\_attn.q\_layernorm.bias', 'language\_model.model.layers.25.input\_layernorm.bias', 'language\_model.model.layers.34.self\_attn.dense.weight', 'language\_model.model.layers.34.input\_layernorm.weight', 'language\_model.model.layers.5.self\_attn.k\_layernorm.bias', 'language\_model.model.layers.2.mlp.dense\_4h\_to\_h.weight', 'language\_model.model.layers.11.self\_attn.dense.bias', 'language\_model.model.layers.17.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.13.mlp.dense\_4h\_to\_h.bias', 'language\_model.model.layers.21.self\_attn.query\_key\_value.weight', 'language\_model.model.lay
> 207 Saved checkpoint at epoch 1.
> \`\`\`
> 
> I try to set the \`safe\_serialization=False\`, the warning disappear but the saved pytorch\_model.bin only 2MB, comparing to around 18GB originally (using 4.35.0).
> 
> \### Expected behavior
> 
> See above

```py
state_dict = accelerator.get_state_dict(model)
model.save_pretrained(output_path, state_dict=state_dict)

```

---

<div class="post-metadata">

**Author:** ![nypabhishek](https://avatars.discourse-cdn.com/v4/letter/n/e9a140/32.png) [@nypabhishek](https://discuss.huggingface.co/u/nypabhishek)\
**Post date:** [March 25, 2025, 10:17am UTC](https://discuss.huggingface.co/t/molformer-model-training-error/147291/7 "2025-03-25T10:17:36Z")

</div>

okkk thank you very much @John6666