← 返回列表

AI安全--LLaVA 式多模态指令微调实践

高级反反爬 security.blog.csdn.net 作者 成就一亿技术人! 发布 2026-11-04T10:49:22+08:00 入库 2026-11-18T02:57:17

本案例仅作为示例学习哈!

pip install -U "transformers>=4.44" peft datasets accelerate pillow
# 显存不足时建议配合 4bit 量化
pip install bitsandbytes

python运行

{"image": "/data/imgs/0001.jpg", "prompt": "详细描述这张图片", "answer": "图中是一只坐在草地上的橘猫……"}

p

import json, torch
from PIL import Image
from datasets import Dataset
from transformers import AutoProcessor, AutoModelForVision2Seq, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model
from transformers import Trainer, TrainingArguments

MODEL = "llava-hf/llava-1.5-7b-hf"          # 视觉编码器+投影层+LLM 已内置

processor = AutoProcessor.from_pretrained(MODEL)
bnb = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)
model = AutoModelForVision2Seq.from_pretrained(
    MODEL, quantization_config=bnb, torch_dtype=torch.bfloat16, device_map="auto")

# 只训练投影层与 LoRA 增量,冻结其余权重(对应书中“冻结骨干、微调连接器”策略)
lora = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
                  target_modules=["q_proj", "k_proj", "v_proj", "o_proj"])
model = get_peft_model(model, lora)
model.print_trainable_parameters()

def load_data(path):
    rows = [json.loads(x) for x in open(path, encoding="utf-8")]
    return Dataset.from_list(rows)

class Collator:
    def __call__(self, batch):
        imgs = [Image.open(x["image"]).convert("RGB") for x in batch]
        texts = [f"USER: <image>\n{x['prompt']} ASSISTANT:" for x in batch]
        labels = [x["answer"] for x in batch]
        enc = processor(images=imgs, text=texts, padding=True, return_tensors="pt")
        tok = processor.tokenizer(labels, padding=True, return_tensors="pt")
        enc["labels"] = tok["input_ids"].masked_fill(tok["input_ids"] == processor.tokenizer.pad_token_id, -100)
        return enc

args = TrainingArguments(
    output_dir="out-llava-lora",
    num_train_epochs=2, per_device_train_batch_size=1, gradient_accumulation_steps=8,
    learning_rate=2e-4, warmup_ratio=0.03, bf16=True,
    logging_steps=10, save_strategy="epoch", remove_unused_columns=False)

trainer = Trainer(model=model, args=args,
                  train_dataset=load_data("train.jsonl"),
                  data_collator=Collator())
trainer.train()
model.save_pretrained("out-llava-lora/adapter")     # 保存 LoRA 权重
processor.save_pretrained("out-llava-lora/adapter")

python运行

from PIL import Image
import torch
from transformers import AutoProcessor, AutoModelForVision2Seq
from peft import PeftModel

processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
model = AutoModelForVision2Seq.from_pretrained(
    "llava-hf/llava-1.5-7b-hf", torch_dtype=torch.bfloat16, device_map="auto")
model = PeftModel.from_pretrained(model, "out-llava-lora/adapter")

img = Image.open("/data/imgs/test.jpg").convert("RGB")
prompt = "USER: <image>\n这张图里发生了什么? ASSISTANT:"
inputs = processor(images=img, text=prompt, return_tensors="pt").to(model.device, torch.bfloat16)
out = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(out[0], skip_special_tokens=True))

python运行

原文链接:https://security.blog.csdn.net/article/details/170033580