# Why my finetuned model size so small and unable to load

**URL:** <https://discuss.huggingface.co/t/why-my-finetuned-model-size-so-small-and-unable-to-load/96276>\
**Category:** Beginners\
**Created:** [July 9, 2024, 2:32am UTC](https://discuss.huggingface.co/t/why-my-finetuned-model-size-so-small-and-unable-to-load/96276 "2024-07-09T02:32:34Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![worstkid92](https://avatars.discourse-cdn.com/v4/letter/w/ba8739/32.png) [@worstkid92](https://discuss.huggingface.co/u/worstkid92)\
**Post date:** [July 9, 2024, 2:32am UTC](https://discuss.huggingface.co/t/why-my-finetuned-model-size-so-small-and-unable-to-load/96276/1 "2024-07-09T02:32:35Z")

</div>

I am new to pretrain.I use customed dataset,which is of alpaca format and it has about 60K rows and is of 19MB.  
I finetuned with “meta-llama/Llama-2-7b-chat-hf” and the output directory is  
du -sh \*  
4.0K adapter\_config.json  
26M adapter\_model.safetensors  
74M checkpoint-200  
74M checkpoint-250  
74M checkpoint-300  
8.0K README.md  
24K runs  
After mergeing with original model the size is still small.  
The finetune scripts is

```auto
import transformers
import textwrap
from transformers import LlamaTokenizer, LlamaForCausalLM
import os
import sys
from typing import List
 
from peft import (
    LoraConfig,
    get_peft_model,
    get_peft_model_state_dict,
    prepare_model_for_kbit_training ,
)
 
import fire
import torch
from datasets import load_dataset
import pandas as pd

data = load_dataset("json", data_files="/mnt/dataset/website/test.json")

data["train"]
BASE_MODEL = "meta-llama/Llama-2-7b-chat-hf" 

model = LlamaForCausalLM.from_pretrained(
    BASE_MODEL,
    load_in_8bit=True,
    torch_dtype=torch.float16,
    device_map="auto",
)
for param in model.parameters():
    # Check if parameter dtype is Float (float32)
    if param.dtype == torch.float32:
        param.data = param.data.to(torch.float16)
 
tokenizer = LlamaTokenizer.from_pretrained(BASE_MODEL)
 
tokenizer.pad_token_id = (
    0 # unk. we want this to be different from the eos token
)
tokenizer.padding_side = "left"

def generate_prompt(data_point):
    return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. # noqa: E501
### Instruction:
{data_point["instruction"]}
### Input:

### Response:
{data_point["output"]}"""
 
CUTOFF_LEN=2048

def tokenize(prompt, add_eos_token=True):
    result = tokenizer(
        prompt,
        truncation=True,
        max_length=CUTOFF_LEN,
        padding=False,
        return_tensors=None,
    )
    if (
        result["input_ids"][-1] != tokenizer.eos_token_id
        and len(result["input_ids"]) < CUTOFF_LEN
        and add_eos_token
    ):
        result["input_ids"].append(tokenizer.eos_token_id)
        result["attention_mask"].append(1)
 
    result["labels"] = result["input_ids"].copy()
 
    return result
 
def generate_and_tokenize_prompt(data_point):
    full_prompt = generate_prompt(data_point)
    tokenized_full_prompt = tokenize(full_prompt)
    return tokenized_full_prompt

train_val = data["train"].train_test_split(
    test_size=200, shuffle=True, seed=42
)
train_data = (
    train_val["train"].map(generate_and_tokenize_prompt)
)
val_data = (
    train_val["test"].map(generate_and_tokenize_prompt)
)

LORA_R = 8
LORA_ALPHA = 32
LORA_DROPOUT= 0.05
LORA_TARGET_MODULES = [
    "q_proj",
    "v_proj",
]
 
BATCH_SIZE = 128
MICRO_BATCH_SIZE = 8
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
LEARNING_RATE = 3e-4
TRAIN_STEPS = 300
OUTPUT_DIR = "experiments-trr"

model = prepare_model_for_kbit_training(model)
config = LoraConfig(
    r=LORA_R,
    lora_alpha=LORA_ALPHA,
    target_modules=LORA_TARGET_MODULES,
    lora_dropout=LORA_DROPOUT,
    bias="none",
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, config)
model.print_trainable_parameters()

training_arguments = transformers.TrainingArguments(
    per_device_train_batch_size=MICRO_BATCH_SIZE,
    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,
    warmup_steps=100,
    max_steps=TRAIN_STEPS,
    learning_rate=LEARNING_RATE,
    fp16 = not torch.cuda.is_bf16_supported(),
    bf16 = torch.cuda.is_bf16_supported(),
    logging_steps=10,
    optim="adamw_torch",
    evaluation_strategy="steps",
    save_strategy="steps",
    eval_steps=50,
    save_steps=50,
    output_dir=OUTPUT_DIR,
    save_total_limit=3,
    load_best_model_at_end=True,
    report_to="tensorboard"
)

data_collator = transformers.DataCollatorForSeq2Seq(
    tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
)

trainer = transformers.Trainer(
    model=model,
    train_dataset=train_data,
    eval_dataset=val_data,
    args=training_arguments,
    data_collator=data_collator
)
model.config.use_cache = False

 
with torch.autocast("cuda"): 
    trainer.train()
model.save_pretrained(OUTPUT_DIR)

```

and merge scripts is

```auto
from transformers import AutoModelForCausalLM
from peft import PeftModel
from transformers import AutoTokenizer
import torch

OUTPUT_DIR = "/mnt/codes/test/merged1"

base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-chat-hf")
peft_model_id = "/mnt/dataset/website/blog_split_qa/experiments"
model = PeftModel.from_pretrained(base_model, peft_model_id)
model.merge_and_unload()

model.save_pretrained(OUTPUT_DIR)

```

Can anyone help