大模型选型与调优实战

# finetune_lora.py
import torch
from datasets import load_dataset
from transformers import (
    AutoModelForCausalLM, AutoTokenizer,
    TrainingArguments, BitsAndBytesConfig,
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, TaskType

MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct"
OUTPUT_DIR = "./output/qwen-lora"
MAX_SEQ_LEN = 2048

# 1. 加载模型（4-bit 量化）
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME, quantization_config=bnb_config,
    device_map="auto", trust_remote_code=True)

model = prepare_model_for_kbit_training(model)

# 2. 配置 LoRA
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16, lora_alpha=32, lora_dropout=0.05,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                    "gate_proj", "up_proj", "down_proj"],
    bias="none",
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# 输出类似：trainable params: 20M || all params: 7B || trainable%: 0.28%

# 3. 加载数据集
dataset = load_dataset("json", data_files={"train": "data/train.jsonl"})

def format_example(example):
    prompt = f"""<|im_start|>system
你是一个专业的AI助手。<|im_end|>
<|im_start|>user
{example['instruction']}{example.get('input', '')}<|im_end|>
<|im_start|>assistant
{example['output']}<|im_end|>"""
    return {"text": prompt}

dataset = dataset.map(format_example)

def tokenize_function(example):
    result = tokenizer(example["text"], truncation=True,
        max_length=MAX_SEQ_LEN, padding="max_length")
    result["labels"] = result["input_ids"].copy()
    return result

tokenized_dataset = dataset.map(tokenize_function,
    remove_columns=dataset["train"].column_names)

# 4. 训练
from trl import SFTTrainer

training_args = TrainingArguments(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    logging_steps=10,
    save_strategy="steps",
    save_steps=100,
    bf16=True,
    optim="paged_adamw_8bit",
    gradient_checkpointing=True,
)

trainer = SFTTrainer(
    model=model, args=training_args,
    train_dataset=tokenized_dataset["train"],
    tokenizer=tokenizer,
)

trainer.train()
trainer.save_model(OUTPUT_DIR)
tokenizer.save_pretrained(OUTPUT_DIR)

# finetune_dpo.py
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model
from trl import DPOTrainer

MODEL_NAME = "Qwen/Qwen2.5-7B-Instruct"
OUTPUT_DIR = "./output/qwen-dpo"

model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)

lora_config = LoraConfig(r=16, lora_alpha=32, lora_dropout=0.05,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"])
model = get_peft_model(model, lora_config)

# 数据格式：{"prompt": "...", "chosen": "...", "rejected": "..."}
dataset = load_dataset("json", data_files={"train": "data/preference.jsonl"})

training_args = TrainingArguments(
    output_dir=OUTPUT_DIR, num_train_epochs=1,
    per_device_train_batch_size=2, gradient_accumulation_steps=8,
    learning_rate=5e-7, bf16=True, gradient_checkpointing=True)

trainer = DPOTrainer(model=model, args=training_args,
    train_dataset=dataset["train"], tokenizer=tokenizer, beta=0.1)

trainer.train()
trainer.save_model(OUTPUT_DIR)

# AWQ 量化
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model = AutoAWQForCausalLM.from_pretrained("Qwen/Qwen2.5-7B-Instruct")
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct")

quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4}
model.quantize(tokenizer, quant_config=quant_config)
model.save_quantized("./output/qwen-7b-awq")
tokenizer.save_pretrained("./output/qwen-7b-awq")

# 使用 GGUF 量化（llama.cpp 格式）
# python convert.py Qwen/Qwen2.5-7B-Instruct \
#   --outfile qwen-7b-q4_k_m.gguf --outtype q4_k_m

# 使用 vLLM 的投机采样
# 小模型快速生成候选 token，大模型验证
# 可以在不损失质量的情况下提升 2-3 倍推理速度

python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct \
    --speculative-model Qwen/Qwen2.5-1.5B-Instruct \
    --num-speculative-tokens 5 \
    --tensor-parallel-size 1 \
    --port 8000

# cost_analysis.py
daily_requests = 10000
avg_input_tokens = 500
avg_output_tokens = 200
days_per_month = 30

# API 方案（GPT-4o）
api_cost = daily_requests * days_per_month * (
    avg_input_tokens * 2.5/1_000_000 +
    avg_output_tokens * 10.0/1_000_000)
print(f"API 月成本（GPT-4o）: ${api_cost:.2f}")

# 自部署方案（Qwen2.5-72B, 2xA100）
gpu_cost = 2.0 * 2 * 24 * days_per_month * 0.6  # 60% 利用率
print(f"自部署月成本: ${gpu_cost:.2f}")

savings = api_cost - gpu_cost
print(f"月节省: ${savings:.2f}")
print(f"建议: {'自部署更划算' if savings > 0 else '使用 API 更划算'}")

from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained("./output/qwen-lora", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained("./output/qwen-lora")

test_data = load_dataset("json", data_files={"test": "data/test.jsonl"})["test"]
correct = 0
for example in test_data:
    inputs = tokenizer(example["instruction"], return_tensors="pt").to(model.device)
    outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.1)
    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    if example["output"].strip() in response:
        correct += 1
print(f"准确率: {correct/len(test_data):.2%}")