当前位置：首页 > news >正文

第N8周：使用Word2vec实现文本分类

news 2025/6/29 11:49:35

文章目录

一、数据预处理
- 1.加载数据
- 2.构建词典
- 3.生成数据批次和迭代器
二、模型构建
- 1.搭建模型
- 2.初始化模型
- 3.定义训练与评估函数
三、训练模型
- 1.拆分数据集并运行模型
四、总结

🍨 本文为🔗365天深度学习训练营中的学习记录博客
🍖 原作者：K同学啊

一、数据预处理

1.加载数据

import torch
import torch.nn as nn
import torchvision
from torchvision import transforms, datasets
import os, PIL, pathlib, warningswarnings.filterwarnings("ignore")device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device

device(type=‘cpu’)

import pandas as pd # 加载自定义中文数据
train_data = pd.read_csv('train.csv', sep='\t', header=None)
train_data.head()

	0	1
0	还有双鸭山到淮阴的汽车票吗13号的	Travel-Query
1	从这里怎么回家	Travel-Query
2	随便播放一首专辑阁楼里的佛里的歌	Music-Play
3	给看一下墓王之王嘛	FilmTele-Play
4	我想看挑战两把s686打突变团竞的游戏视频	Video-Play

# 构造数据集迭代器
def coustom_data_iter(texts, labels):for x, y in zip(texts, labels):yield x, yx = train_data[0].values[:]
# 多类标签的one-shot展开
y = train_data[1].values[:]

2.构建词典

from gensim.models import Word2Vec
import numpy as np# 训练Word2vec浅层模型
w2v = Word2Vec(vector_size = 100,   # 是指特征向量的维度，默认为100min_count = 3)       # 可以对字典做截断，词频少于min_count次数的单词会被丢弃掉，默认值为5w2v.build_vocab(x)
w2v.train(x,total_examples=w2v.corpus_count,epochs=20)

(2732785, 3663560)

Word2Vec可以直接训练模型，一步到位。这里分了三步
●第一步构建一个空模型
●第二步使用 build_vocab 方法根据输入的文本数据 x 构建词典。build_vocab 方法会统计输入文本中每个词汇出现的次数，并按照词频从高到低的顺序将词汇加入词典中。
●第三步使用 train 方法对模型进行训练，total_examples 参数指定了训练时使用的文本数量，这里使用的是 w2v.corpus_count 属性，表示输入文本的数量
如果一步到位的话代码为:

w2v = Word2Vec(x, vector_size=100, min_count=3, epochs=20)

# 将文本转化为向量
def average_vec(text):vec = np.zeros(100).reshape((1,100))for word in text:try:vec += w2v.wv[word].reshape((1,100))except KeyError:continuereturn vec# 将词向量保存为Ndarray
x_vec = np.concatenate([average_vec(z) for z in x])# 保存Word2Vec模型及词向量
w2v.save('w2v_model.pkl')

train_iter = coustom_data_iter(x_vec, y)
len(x), len(x_vec)

(12100, 12100)

label_name = list(set(train_data[1].values[:]))
print(label_name)

[‘Travel-Query’, ‘Radio-Listen’, ‘Alarm-Update’, ‘FilmTele-Play’, ‘TVProgram-Play’, ‘HomeAppliance-Control’, ‘Calendar-Query’, ‘Audio-Play’, ‘Video-Play’, ‘Other’, ‘Music-Play’, ‘Weather-Query’]

3.生成数据批次和迭代器

text_pipeline = lambda x: average_vec(x)
label_pipeline = lambda x: label_name.index(x)

text_pipeline("你在干嘛")

array([[ 0.44942591, 0.37034334, 0.82435736, 0.57583929, -2.19971114,
-0.26266199, 1.54612615, 0.86057729, 0.94607782, -0.56024504,
-1.2855403 , -3.96268934, 1.00411272, -0.78717487, 0.11495599,
1.7602468 , 2.57005858, -2.04502518, 4.77852516, -1.15009709,
2.75658896, -0.7439712 , -0.50604325, 0.23402849, -0.85734205,
-0.64015828, -1.63281712, -1.22751366, 2.32347407, -2.94733901,
1.86662954, 1.20093471, -0.22566201, -0.02635491, 1.06643996,
0.17282215, -0.57236505, 3.87719914, -2.36707568, 1.28222315,
0.16626818, 0.52857486, 0.2673108 , 1.32945235, -0.51124085,
0.68514908, 0.87900299, -0.9519761 , -2.69660458, 1.78133809,
-0.16500359, -2.11181024, -1.16635181, 1.22090494, -0.76275884,
-0.01114198, 0.42615444, -1.23754779, 0.07603779, -0.04253516,
1.32692097, -1.66303211, 2.16462026, -0.9799156 , -0.9070952 ,
0.87778991, -1.08169729, 0.92559687, 0.64850095, 0.20967194,
0.26563513, 1.03787032, 2.3587795 , -0.7511736 , 0.74099658,
-0.15902402, -2.69873536, 0.13621271, 1.08319706, -0.18128317,
-1.8476568 , -0.67964274, -2.43600948, 2.98213428, -1.72624808,
-0.87052085, 2.28517788, -1.87188464, -0.26412555, -0.37503278,
1.51758769, -1.25159131, 0.87080194, 0.85611653, 0.85986885,
-0.60930844, -0.11496616, 0.66294981, -2.06530389, 0.11790894]])

label_pipeline("Travel-Query")

from torch.utils.data import DataLoaderdef collate_batch(batch):label_list, text_list= [], []for (_text, _label) in batch:# 标签列表label_list.append(label_pipeline(_label))# 文本列表processed_text = torch.tensor(text_pipeline(_text), dtype=torch.float32)text_list.append(processed_text)label_list = torch.tensor(label_list, dtype=torch.int64)text_list = torch.cat(text_list)return text_list.to(device), label_list.to(device)# 数据加载器，调用示例
dataloader = DataLoader(train_iter,batch_size=8,shuffle=False,collate_fn=collate_batch)

二、模型构建

1.搭建模型

from torch import nnclass TextClassificationModel(nn.Module):def __init__(self, num_class):super(TextClassificationModel, self).__init__()self.fc = nn.Linear(100, num_class)def forward(self, text):return self.fc(text)

2.初始化模型

num_class = len(label_name)
vocab_size = 100000
em_size = 12
model = TextClassificationModel(num_class).to(device)

3.定义训练与评估函数

import timedef train(dataloader):model.train() # 切换到训练模式total_acc, train_loss, total_count = 0, 0, 0log_interval = 50start_time = time.time()for idx, (text, label) in enumerate(dataloader):predicted_label = model(text)optimizer.zero_grad() # grad属性归零loss = criterion(predicted_label, label) # 计算网络输出和真实值之间的差距，label为真实值loss.backward() # 反向传播torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1) # 梯度裁剪optimizer.step() # 每一步自东更新# 记录acc与losstotal_acc += (predicted_label.argmax(1) == label).sum().item()train_loss += loss.item()total_count += label.size(0)if idx % log_interval == 0 and idx > 0:elapsed = time.time() - start_timeprint('| epoch {:1d} | {:4d}/{:4d} batches''| train_acc {:4.3f} train_loss {:4.5f}'.format(epoch, idx, len(dataloader),total_acc/total_count, train_loss/total_count))total_acc, train_loss, total_count = 0, 0, 0start_time = time.time()def evaluate(dataloader):model.eval() # 切换为测试模式total_acc, train_loss, total_count = 0, 0, 0with torch.no_grad():for idx, (text,label) in enumerate(dataloader):predicted_label = model(text)loss = criterion(predicted_label, label) # 计算loss值# 记录测试数据total_acc += (predicted_label.argmax(1) == label).sum().item()train_loss += loss.item()total_count += label.size(0)return total_acc/total_count, train_loss/total_count

三、训练模型

1.拆分数据集并运行模型

from torch.utils.data.dataset import random_split
from torchtext.data.functional import to_map_style_dataset# 超参数
EPOCHS = 10 
LR = 5
BATCH_SIZE = 64criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=LR)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.1)
total_accu = None# 构建数据集
train_iter = coustom_data_iter(train_data[0].values[:], train_data[1].values[:])
train_dataset = to_map_style_dataset(train_iter)split_train_, split_valid_ = random_split(train_dataset,[int(len(train_dataset)*0.8), int(len(train_dataset)*0.2)])train_dataloader = DataLoader(split_train_, batch_size=BATCH_SIZE,shuffle=True, collate_fn=collate_batch)valid_dataloader = DataLoader(split_valid_, batch_size=BATCH_SIZE,shuffle=True, collate_fn=collate_batch)for epoch in range(1, EPOCHS + 1):epoch_start_time = time.time()train(train_dataloader)val_acc, val_loss = evaluate(valid_dataloader)# 获取当前的学习率lr = optimizer.state_dict()['param_groups'][0]['lr']if total_accu is not None and total_accu > val_acc:scheduler.step()else:total_accu = val_accprint('-' * 69)print('| epoch {:1d} | time: {:4.2f}s | ''valid_acc {:4.3f} valid_loss {:4.3f} | lr {:4.6f}'.format(epoch,time.time() - epoch_start_time,val_acc,val_loss,lr))print('-' * 69)

test_acc, test_loss = evaluate(valid_dataloader)
print('模型准确率为：{:5.4f}'.format(test_acc))

模型准确率为：0.8876

def predict(text, text_pipeline):with torch.no_grad():text = torch.tensor(text_pipeline(text), dtype=torch.float32)print(text.shape)output = model(text)return output.argmax(1).item()# ex_text_str = "随便播放一首专辑阁楼里的佛里的歌"
ex_text_str = "还有双鸭山到淮阴的汽车票吗13号的"model = model.to("cpu")print("该文本的类别是：%s" %label_name[predict(ex_text_str, text_pipeline)])