以下代码实现了LoRA(低秩适配)微调层,请阅读代码并填写空缺部分。[1] 和 [2] 处应分别填入:
import torch
import torch.nn as nn
import math
class LoRALinear(nn.Module):
"""在原始线性层旁添加低秩分解矩阵实现LoRA微调"""
def __init__(self, original_linear: nn.Linear, rank: int = 8, alpha: float = 16.0):
super().__init__()
self.original_linear = original_linear
self.rank = rank
self.alpha = alpha
self.scaling = alpha / rank
in_features = original_linear.in_features
out_features = original_linear.out_features
# [1] 初始化低秩矩阵 A 和 B
# LoRA 的核心:ΔW = A @ B,其中 A: (in_features, rank),B: (rank, out_features)
[1]_____________________________________________________________________
[2]_____________________________________________________________________
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
nn.init.zeros_(self.lora_B)
# 冻结原始权重
self.original_linear.weight.requires_grad = False
if self.original_linear.bias is not None:
self.original_linear.bias.requires_grad = False
def forward(self, x):
# 原始线性层的输出
original_output = self.original_linear(x)
# [2] 计算 LoRA 的增量输出:x 先经过 A 再经过 B,并乘以缩放因子
[1] self.lora_A = nn.Parameter(torch.empty(in_features, rank)) 和 self.lora_B = nn.Parameter(torch.empty(rank, out_features))
[2] lora_output = (x @ self.lora_A@ self.lora_B) * self.scaling
[1] self.lora_A = nn.Parameter(torch.empty(out_features, rank))和 self.lora_B = nn.Parameter(torch.empty(rank, in_features))
[2] lora_output = (x @ self.lora_A@ self.lora_B) * self.scaling
[1] self.lora_A = nn.Parameter(torch.empty(in_features, rank))和 self.lora_B = nn.Parameter(torch.empty(rank, out_features))
[2] lora_output = (x + self.lora_A + self.lora_B) * self.scaling
[1] self.lora_A= torch.empty(in_features, rank) 和self.lora_B = torch.empty(rank, out_features)
[2] lora_output = (x @ self.lora_A@ self.lora_B) * self.scaling