本案例仅作为示例学习哈!
pip install -U "transformers>=4.44" peft datasets accelerate pillow
# 显存不足时建议配合 4bit 量化
pip install bitsandbytes
python运行
- 1
- 2
- 3
{"image": "/data/imgs/0001.jpg", "prompt": "详细描述这张图片", "answer": "图中是一只坐在草地上的橘猫……"}
p
- 1
import json, torch
from PIL import Image
from datasets import Dataset
from transformers import AutoProcessor, AutoModelForVision2Seq, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from transformers import Trainer, TrainingArguments
MODEL = "llava-hf/llava-1.5-7b-hf" # 视觉编码器+投影层+LLM 已内置
processor = AutoProcessor.from_pretrained(MODEL)
bnb = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)
model = AutoModelForVision2Seq.from_pretrained(
MODEL, quantization_config=bnb, torch_dtype=torch.bfloat16, device_map="auto")
# 只训练投影层与 LoRA 增量,冻结其余权重(对应书中“冻结骨干、微调连接器”策略)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
target_modules=["q_proj", "k_proj", "v_proj", "o_proj"])
model = get_peft_model(model, lora)
model.print_trainable_parameters()
def load_data(path):
rows = [json.loads(x) for x in open(path, encoding="utf-8")]
return Dataset.from_list(rows)
class Collator:
def __call__(self, batch):
imgs = [Image.open(x["image"]).convert("RGB") for x in batch]
texts = [f"USER: <image>\n{x['prompt']} ASSISTANT:" for x in batch]
labels = [x["answer"] for x in batch]
enc = processor(images=imgs, text=texts, padding=True, return_tensors="pt")
tok = processor.tokenizer(labels, padding=True, return_tensors="pt")
enc["labels"] = tok["input_ids"].masked_fill(tok["input_ids"] == processor.tokenizer.pad_token_id, -100)
return enc
args = TrainingArguments(
output_dir="out-llava-lora",
num_train_epochs=2, per_device_train_batch_size=1, gradient_accumulation_steps=8,
learning_rate=2e-4, warmup_ratio=0.03, bf16=True,
logging_steps=10, save_strategy="epoch", remove_unused_columns=False)
trainer = Trainer(model=model, args=args,
train_dataset=load_data("train.jsonl"),
data_collator=Collator())
trainer.train()
model.save_pretrained("out-llava-lora/adapter") # 保存 LoRA 权重
processor.save_pretrained("out-llava-lora/adapter")
python运行
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15
- 16
- 17
- 18
- 19
- 20
- 21
- 22
- 23
- 24
- 25
- 26
- 27
- 28
- 29
- 30
- 31
- 32
- 33
- 34
- 35
- 36
- 37
- 38
- 39
- 40
- 41
- 42
- 43
- 44
- 45
- 46
from PIL import Image
import torch
from transformers import AutoProcessor, AutoModelForVision2Seq
from peft import PeftModel
processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
model = AutoModelForVision2Seq.from_pretrained(
"llava-hf/llava-1.5-7b-hf", torch_dtype=torch.bfloat16, device_map="auto")
model = PeftModel.from_pretrained(model, "out-llava-lora/adapter")
img = Image.open("/data/imgs/test.jpg").convert("RGB")
prompt = "USER: <image>\n这张图里发生了什么? ASSISTANT:"
inputs = processor(images=img, text=prompt, return_tensors="pt").to(model.device, torch.bfloat16)
out = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(out[0], skip_special_tokens=True))
python运行
- 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
- 10
- 11
- 12
- 13
- 14
- 15