# RuntimeError: grad can be implicitly created only for scalar outputs

**URL:** <https://discuss.huggingface.co/t/runtimeerror-grad-can-be-implicitly-created-only-for-scalar-outputs/50282>\
**Category:** 🤗Transformers\
**Created:** [August 10, 2023, 1:31pm UTC](https://discuss.huggingface.co/t/runtimeerror-grad-can-be-implicitly-created-only-for-scalar-outputs/50282 "2023-08-10T13:31:46Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![Luan77777](https://avatars.discourse-cdn.com/v4/letter/l/ecc23a/32.png) [@Luan77777](https://discuss.huggingface.co/u/Luan77777)\
**Post date:** [August 10, 2023, 1:31pm UTC](https://discuss.huggingface.co/t/runtimeerror-grad-can-be-implicitly-created-only-for-scalar-outputs/50282/1 "2023-08-10T13:31:46Z")

</div>

Hi! I am currently trying to train a Speech2Text model from scratch. I have everything set up and my dataset matches the requirements of the documentation in terms of the tensor shapes! I dont know why i get this error is there something wrong with my data or the model config?? This is my Code and the whole error message:

import sys  
from datasets import load\_dataset\_builder, load\_dataset, dataset\_dict  
from transformers import Seq2SeqTrainer, Speech2TextForConditionalGeneration, Speech2TextConfig, Speech2TextProcessor,   
Speech2TextTokenizer, Speech2TextFeatureExtractor, trainer\_utils,   
Trainer, DataCollatorWithPadding, Speech2TextModel, TrainingArguments, Seq2SeqTrainingArguments  
import torch  
from sklearn.model\_selection import train\_test\_split  
import soundfile as sf  
from dataclasses import dataclass, field  
from typing import Any, Dict, List, Optional, Union  
import torchaudio  
import numpy as np  
import json

test\_dataset = load\_dataset(‘audiofolder’, data\_dir=r"C:\Users\LuanL\Desktop\Dataset Test")  
test\_dataset = test\_dataset[‘train’]  
print(test\_dataset)

tokenizer = Speech2TextTokenizer(spm\_file=r"C:\Users\LuanL\Desktop\BA\Tokenizer\char.model",  
unk\_token=‘’, eos\_token=‘’, sos\_token=‘’,  
vocab\_file=r"C:\Users\LuanL\Desktop\BA\Tokenizer\char.json", pad\_token=‘’)

feature\_extractor = Speech2TextFeatureExtractor(sampling\_rate=16000, num\_mel\_bins=80, feature\_size=80)

processor = Speech2TextProcessor(feature\_extractor=feature\_extractor, tokenizer=tokenizer)

def preprocess\_function(examples):  
audio\_arrays = [x[“array”] for x in examples[“audio”]]

```
inputs_audio = processor(

    audio_arrays,
    sampling_rate=16000,
    padding=True,
    max_length=100000000,
    truncation=True,

)

print(inputs_audio)

return inputs_audio

```

def preprocess\_function\_text(examples):  
text = [x for x in examples[‘transcription’]]

```
input_text = tokenizer(text, padding=True, truncation=True, return_tensors="np")

input_ids_list = []
attention_mask_list = []

for i in range(len(text)):
    input_ids = input_text.input_ids[i]
    attention_mask = input_text.attention_mask[i]

    input_ids_list.append(input_ids)
    attention_mask_list.append(attention_mask)

return {
    'decoder_input_ids': input_ids_list,
    'decoder_attention_mask': attention_mask_list
}

```

test\_dataset = test\_dataset.map(preprocess\_function, batched=True)  
test\_dataset = test\_dataset.map(preprocess\_function\_text, batched=True)  
test\_dataset = test\_dataset.remove\_columns([‘audio’, ‘transcription’])  
test\_dataset.set\_format(type=“pt”)

training\_args = Seq2SeqTrainingArguments(  
evaluation\_strategy=“epoch”,  
output\_dir=r"C:\Users\LuanL\Desktop\BA\Train Loop",  
num\_train\_epochs=3,  
learning\_rate=2e-5,  
# per\_device\_eval\_batch\_size=4,  
weight\_decay=0.01,  
logging\_dir=r"C:\Users\LuanL\Desktop\BA\Train Loop\Log",  
gradient\_accumulation\_steps=1,  
max\_grad\_norm=1.0,  
)

config = Speech2TextConfig(  
return\_dict=False,  
sampling\_rate=16000,  
vocab\_size=tokenizer.vocab\_size,  
use\_cache=False,  
activation\_function=‘relu’,  
max\_target\_positions=2048,  
max\_source\_positions=100000,  
input\_feat\_per\_channel=80,  
pad\_token\_id=35,  
eos\_token\_id=2,  
bos\_token\_id=1,

)

model = Speech2TextModel(config)  
model.train()

trainer = Seq2SeqTrainer(  
model=model,  
args=training\_args,  
train\_dataset=test\_dataset,  
)

trainer.train()

Error message:  
C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\Scripts\python.exe C:\Users\LuanL\PycharmProjects\Speech2TextNeu\main.py  
Dataset({  
features: [‘audio’, ‘transcription’],  
num\_rows: 1  
})  
Map: 0%| | 0/1 [00:00\<?, ? examples/s]Asking to truncate to max\_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.  
Map: 100%|██████████| 1/1 [00:00\<00:00, 45.45 examples/s]  
Dataset({  
features: [‘input\_features’, ‘attention\_mask’, ‘decoder\_input\_ids’, ‘decoder\_attention\_mask’],  
num\_rows: 1  
})  
C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\transformers\optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning  
warnings.warn(  
0%| | 0/3 [00:00\<?, ?it/s]Traceback (most recent call last):  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\main.py”, line 175, in   
trainer.train()  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\transformers\trainer.py”, line 1539, in train  
return inner\_training\_loop(  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\transformers\trainer.py”, line 1809, in _inner\_training\_loop  
tr\_loss\_step = self.training\_step(model, inputs)  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\transformers\trainer.py”, line 2665, in training\_step  
self.accelerator.backward(loss)  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\accelerate\accelerator.py”, line 1853, in backward  
loss.backward(\*\*kwargs)  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\torch\_tensor.py”, line 487, in backward  
torch.autograd.backward(  
File "C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\torch\autograd\_init_.py", line 193, in backward  
grad\_tensors\_ = _make\_grads(tensors, grad\_tensors_, is\_grads\_batched=False)  
File “C:\Users\LuanL\PycharmProjects\Speech2TextNeu\venv\lib\site-packages\torch\autograd\__init_\_.py”, line 88, in \_make\_grads  
raise RuntimeError(“grad can be implicitly created only for scalar outputs”)  
RuntimeError: grad can be implicitly created only for scalar outputs  
0%| | 0/3 [00:02\<?, ?it/s]

Process finished with exit code 1