4.权重衰减(weight decay)

4.1 手动实现权重衰减

import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):X=torch.normal(0,1,size=(num_inputs,w.shape[0]))y=X@w+by+=torch.normal(0,0.1,size=y.shape)return X,y
def load_array(data,batch_size,is_train=True):dataset=TensorDataset(*data)return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)b=torch.zeros(1,requires_grad=True)return [w,b]
def l2_penalty(w):return 0.5*torch.sum(w.pow(2))def linear_reg(X,w,b):return torch.matmul(X,w)+b
def mse_loss(y_hat,y):return (y_hat-y)**2/2
def sgd(params,lr,batch_size):for params in params:params.data-=lr*params.grad/batch_sizeparams.grad.zero_()
def evaluate_loss(net, data_iter, loss):total_loss, total_samples = 0.0, 0for X, y in data_iter:l = loss(net(X), y)total_loss += l.sum().item()total_samples += y.numel()return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=10,0.05,3
#animator=SimpleAnimator()
for epoch in range(num_epochs):for X,y in train_iter:l=loss(net(X),y)+lambd*l2_penalty(w)l.sum().backward()sgd([w,b],lr,batch_size)if (epoch+1)%5==0:train_loss=evaluate_loss(net,train_iter,loss)test_loss=evaluate_loss(net,test_iter,loss)#animator.add(epoch+1,train_loss,test_loss)print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()

4.2 简单实现权重衰减

import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):X=torch.normal(0,1,size=(num_inputs,w.shape[0]))y=X@w+by+=torch.normal(0,0.1,size=y.shape)return X,y
def load_array(data,batch_size,is_train=True):dataset=TensorDataset(*data)return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)b=torch.zeros(1,requires_grad=True)return [w,b]
def l2_penalty(w):return 0.5*torch.sum(w.pow(2))
def linear_reg(X,w,b):return torch.matmul(X,w)+b
def mse_loss(y_hat,y):return ((y_hat-y)**2).sum()/2
def evaluate_loss(net, data_iter, loss):total_loss, total_samples = 0.0, 0for X, y in data_iter:l = loss(net(X), y)total_loss += l.item()*y.shape[0]total_samples += y.numel()return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=100,0.001,3
optimizer=torch.optim.SGD([w,b],lr=lr,weight_decay=0.001)
#animator=SimpleAnimator()
for epoch in range(num_epochs):for X,y in train_iter:optimizer.zero_grad()l=loss(net(X),y)l.backward()#sgd([w,b],lr,batch_size)optimizer.step() if (epoch+1)%5==0:train_loss=evaluate_loss(net,train_iter,loss)test_loss=evaluate_loss(net,test_iter,loss)#animator.add(epoch+1,train_loss,test_loss)print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()