基于OPUS-MT模型的中译英程序实现

import osimport zipfilefrom urllib.request import urlretrievefrom tqdm import tqdmclass TqdmUpTo(tqdm):def update_to(self, b=1, bsize=1, tsize=None):if tsize is not None:self.total = tsizeself.update(b * bsize - self.n)url = "https://object.pouta.csc.fi/OPUS-CCAligned/v1/moses/en-zh_CN.txt.zip"save_path = "en-zh.txt.zip"extract_dir = "/kaggle/working/CC"os.makedirs(extract_dir, exist_ok=True)if not os.path.exists(save_path):print(f"开始从 {url} 下载文件...")with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:urlretrieve(url, save_path, reporthook=t.update_to)print(f"下载完成并保存到 {save_path}")else:print(f"文件已存在: {save_path}")print(f"开始解压文件到 {extract_dir}...")with zipfile.ZipFile(save_path, 'r') as zip_ref:zip_ref.extractall(extract_dir)print(f"文件已解压到：{extract_dir}")

import osimport reimport pandas as pddef read_lines(file_path):with open(file_path, "r", encoding="utf-8") as f:return [line.strip() for line in f if line.strip()]en_path = "/kaggle/working/CC/CCAligned.en-zh_CN.en"zh_path = "/kaggle/working/CC/CCAligned.en-zh_CN.zh_CN"en_lines = read_lines(en_path)zh_lines = read_lines(zh_path)min_len = min(len(en_lines), len(zh_lines))en_lines = en_lines[:min_len]zh_lines = zh_lines[:min_len]df = pd.DataFrame({"en": en_lines, "zh": zh_lines})

def find_extreme_sentences(column, length_column):min_idx = df[length_column].idxmin()max_idx = df[length_column].idxmax()min_sentence = df.loc[min_idx, column]max_sentence = df.loc[max_idx, column]min_length = df.loc[min_idx, length_column]max_length = df.loc[max_idx, length_column]return min_sentence, min_length, max_sentence, max_lengthdf['en_length'] = df['en'].apply(lambda x: len(x.split())) df['zh_length'] = df['zh'].apply(lambda x: len(x))        print("数据基本统计：")print(f"英文句子平均长度: {df['en_length'].mean():.2f}")print(f"中文句子平均长度: {df['zh_length'].mean():.2f}")min_en, min_en_len, max_en, max_en_len = find_extreme_sentences('en', 'en_length')min_zh, min_zh_len, max_zh, max_zh_len = find_extreme_sentences('zh', 'zh_length')print("\n最短和最长句子：")print(f"最短英文句子({min_en_len}词): {min_en}")print(f"最长英文句子({max_en_len}词): {max_en}")print(f"最短中文句子({min_zh_len}字符): {min_zh}")print(f"最长中文句子({max_zh_len}字符): {max_zh}")print("\n结果验证：")print(f"英文最短句子实际分词验证: {len(min_en.split())}")print(f"英文最长句子实际分词验证: {len(max_en.split())}")print(f"中文最短句子实际长度验证: {len(min_zh)}")
print(f"中文最长句子实际长度验证: {len(max_zh)}")

import matplotlib.pyplot as pltimport seaborn as snsdef plot_length_distribution(df,xlim):xlim=xlimplt.figure(figsize=(10, 6))sns.histplot(df['en_length'], kde=True, bins=30,color='#99ffff', alpha=0.6, label='English (Words)')sns.histplot(df['zh_length'], kde=True, bins=30,color='#99b3ff', alpha=0.6, label='Chinese (Characters)')max_len = max(df['en_length'].max(), df['zh_length'].max())plt.xlim(0, max_len * 1.1)plt.xlim(0,xlim)plt.title('Sentence Length Distribution', fontsize=14, fontweight='bold')plt.xlabel('Length', fontsize=12)plt.ylabel('Frequency', fontsize=12)plt.legend(fontsize=12)plt.tight_layout()plt.show()plot_length_distribution(df,600)

from scipy.stats import skewdef analyze_skewness(column):skewness = skew(column, nan_policy='omit')if skewness > 1:return f"高度右偏态（偏度值: {skewness:.2f}）"elif 0.5 <= skewness <= 1:return f"中度右偏态（偏度值: {skewness:.2f}）"elif skewness < -1:return f"高度左偏态（偏度值: {skewness:.2f}）"elif -1 <= skewness <= -0.5:return f"中度左偏态（偏度值: {skewness:.2f}）"else:return f"近似正态分布（偏度值: {skewness:.2f}）"

def filter_by_length(df,en_min_words=3, en_max_words=50, zh_min_chars=3, zh_max_chars=150，min_ratio=0.5,  max_ratio=2.0,  enforce_ratio=True):if not all(col in df.columns for col in ['en_length', 'zh_length']):raise ValueError("DataFrame必须包含'en_length'和'zh_length'列")if en_min_words >= en_max_words:raise ValueError("en_min_words必须小于en_max_words")if zh_min_chars >= zh_max_chars:raise ValueError("zh_min_chars必须小于zh_max_chars")ratio = df['zh_length'] / df['en_length'].replace(0, 1e-6)mask = ((df['en_length'] >= en_min_words) &(df['en_length'] <= en_max_words) &(df['zh_length'] >= zh_min_chars) &(df['zh_length'] <= zh_max_chars))if enforce_ratio:mask &= (ratio >= min_ratio) & (ratio <= max_ratio)return df.loc[mask].copy()

def clean_duplicates(df):df = df.dropna(subset=['en', 'zh']) df = df[(df['en'] != "") & (df['zh'] != "")] df = df.drop_duplicates(subset=['en', 'zh']) return df

import redef clean_text(text, lang):text = re.sub(r'<[^>]+>', '', text) text = re.sub(r'&[a-z]+;', '', text) if lang == 'zh':text = re.sub(r'[^\u4e00-\u9fa5\w\s,.!?，。！？]', '', text)text = re.sub(r'\s+', ' ', text)elif lang == 'en':text = re.sub(r'[^\w\s,.!?]', '', text)text = re.sub(r'\s+', ' ', text)return text.strip()

def clean_mixed_language(df, en_col='en', zh_col='zh',chinese_threshold=0.2, english_threshold=0.1):df_clean = df.copy()to_drop = set()for idx, row in df_clean.iterrows():en_text = str(row[en_col])zh_in_en = len(re.findall(r'[\u4e00-\u9fa5]', en_text))en_length = len(en_text)zh_ratio_in_en = zh_in_en / en_length if en_length > 0 else 0zh_text = str(row[zh_col])en_in_zh = len(re.findall(r'[a-zA-Z]', zh_text))zh_length = len(zh_text)en_ratio_in_zh = en_in_zh / zh_length if zh_length > 0 else 0if (zh_ratio_in_en > chinese_threshold) or (en_ratio_in_zh > english_threshold):to_drop.add(idx)# 成对删除混杂的句子df_clean = df_clean.drop(index=to_drop).reset_index(drop=True)print(f"原始数据量: {len(df)}")print(f"删除混杂句子数量: {len(to_drop)}")print(f"清理后数据量: {len(df_clean)}")print(f"中英文句子数量保持: {len(df_clean[en_col])} == {len(df_clean[zh_col])}")return df_clean

print('截断过长、过短句子')df=filter_by_length(df,en_min_words=1,en_max_words=100, zh_min_chars=1,zh_max_chars=150,min_ratio=0.67, max_ratio=1.5  )print(df.shape)print("过滤空行和重复句对...")df = clean_duplicates(df)print(df.shape)print("开始清洗中文文本...")df['zh'] = df['zh'].apply(lambda x: clean_text(x, 'zh'))print("开始清洗英文文本...")df['en'] = df['en'].apply(lambda x: clean_text(x, 'en'))print(df.shape)print("检查中英句子混杂")df=clean_mixed_language(df, en_col='en', zh_col='zh',chinese_threshold=0.2, english_threshold=0.1)print(df.shape)print(f"清洗后剩余数据形状：{df.shape} 条")print("清洗后前5条示例：")print(df.head())

plt.figure(figsize=(10, 6))plt.scatter(df['en_length'], df['zh_length'], c='#99b3ff', alpha=0.6, s=100, edgecolors='w', linewidth=1)plt.title('The correlation of sequence lengths', fontsize=14, pad=20,fontweight='bold')plt.xlabel('Length of English', fontsize=12)plt.ylabel('Length of Chinese', fontsize=12)plt.grid(True, linestyle='--', alpha=0.6)plt.show()

import pandas as pdfrom wordcloud import WordCloudimport matplotlib.pyplot as pltfrom nltk.corpus import stopwordsfrom collections import Counterimport nltknltk.download('stopwords')def generate_english_wordcloud(df, text_col='en', max_words=200, custom_stopwords=None):stop_words = set(stopwords.words('english'))stop_words.update(["'s", "n't", "'m", "'re", "'ve"])  # 追加英文缩写if custom_stopwords:stop_words.update(custom_stopwords)text = ' '.join(df[text_col].dropna().astype(str))words = [word.lower() for word in text.split()if (word.lower() not in stop_words andlen(word) > 2 andword.isalpha())]wc = WordCloud(width=800,height=600,background_color='white',max_words=max_words,colormap='viridis',collocations=False).generate(' '.join(words))plt.figure(figsize=(10, 6))plt.imshow(wc, interpolation='bilinear')plt.axis('off')plt.title("English Word Cloud ", fontsize=18, pad=20,fontweight='bold')plt.tight_layout()plt.show()top_words = Counter(words).most_common(10)print("Top 10高频词示例：")print(top_words)generate_english_wordcloud(df)

from torch.utils.data import Datasetimport torchclass TranslationDataset(Dataset):def __init__(self, df, tokenizer, max_length=128):self.df = dfself.tokenizer = tokenizerself.max_length = max_lengthdef __len__(self):return len(self.df)def __getitem__(self, idx):# 获取源语言（中文）和目标语言（英文）句子source_text = self.df.iloc[idx]['zh']target_text = self.df.iloc[idx]['en']# 对源语言和目标语言进行编码encoding = self.tokenizer(source_text,padding='max_length',truncation=True,max_length=self.max_length,return_tensors='pt')# 对目标语言进行编码（注意：这里使用target_text作为labels）target_encoding = self.tokenizer(target_text,padding='max_length',truncation=True,max_length=self.max_length,return_tensors='pt')# 准备模型输入inputs = {'input_ids': encoding['input_ids'].flatten(),'attention_mask': encoding['attention_mask'].flatten(),'labels': target_encoding['input_ids'].flatten()}return inputs

from sklearn.model_selection import train_test_splittrain_df,temp_df= train_test_split(df, test_size=0.2, random_state=42)val_df,test_df=train_test_split(temp_df,test_size=0.5,random_state=42)

!pip uninstall -y transformers tokenizers sentencepiece!pip install transformers==4.37.0 tokenizers sentencepiece

from torch.utils.data import Dataset, DataLoaderfrom transformers import AutoTokenizer,AutoModelForSeq2SeqLMtokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-zh-en")model = AutoModelForSeq2SeqLM.from_pretrained("Helsinki-NLP/opus-mt-zh-en")

train_dataset = TranslationDataset(train_df, tokenizer)val_dataset = TranslationDataset(val_df, tokenizer)test_dataset = TranslationDataset(test_df, tokenizer)

batch_size = 32train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)val_dataloader = DataLoader(val_dataset, batch_size=batch_size)test_dataloader = DataLoader(test_dataset, batch_size=batch_size)sample_batch = next(iter(train_dataloader))print('数据加载器测试:')print(f"输入ID形状: {sample_batch['input_ids'].shape}")print(f"注意力掩码形状: {sample_batch['attention_mask'].shape}")print(f"标签形状: {sample_batch['labels'].shape}")

from torch import nnfrom torch.optim import Optimizerfrom torch.optim.lr_scheduler import _LRSchedulerfrom torch.cuda.amp import GradScalerfrom tqdm import tqdmdef train_epoch(model, dataloader, optimizer: Optimizer, scheduler: _LRScheduler, device, epoch, scaler: GradScaler, gradient_accumulation_steps=1):model.train()total_loss = 0.0progress_bar = tqdm(enumerate(dataloader), total=len(dataloader), desc=f'Epoch {epoch+1}')for step, batch in progress_bar:# 数据移动至设备（显式转换，避免隐式拷贝）input_ids = batch['input_ids'].to(device, non_blocking=True)attention_mask = batch['attention_mask'].to(device, non_blocking=True)labels = batch['labels'].to(device, non_blocking=True)# 混合精度前向传播（显式上下文管理器）with torch.amp.autocast(device_type='cuda'):  # 更新为新的 autocast 调用方式outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)# 梯度累积：计算当前批次平均损失loss = outputs.loss / gradient_accumulation_steps# 梯度缩放与反向传播scaler.scale(loss).backward()total_loss += loss.item()  # 记录原始损失（非平均后的值，便于后续分析）# 梯度更新条件（使用数学取模，逻辑更清晰）if (step + 1) % gradient_accumulation_steps == 0:scaler.step(optimizer)   # 混合精度更新参数scaler.update()          # 更新缩放因子optimizer.zero_grad(set_to_none=True)  # 更高效的梯度清零方式scheduler.step()         # 基于步数的调度器更新# 清空进度条缓存（避免显存泄漏）progress_bar.reset()# 返回 epoch 平均损失（总损失/总批次，而非总样本数）avg_loss = total_loss / len(dataloader)return avg_loss

from tqdm import tqdmfrom transformers import AdamW, get_linear_schedule_with_warmupfrom torch.cuda.amp import GradScaler, autocastimport numpy as npfrom torch.utils.tensorboard import SummaryWriter# 设置设备device = torch.device("cuda" if torch.cuda.is_available() else "cpu")model.to(device)print(f"使用设备: {device}")# 优化器optimizer = AdamW(model.parameters(), lr=5e-5)# 训练总步数和预热步数num_epochs = 10 # 增加训练轮数total_steps = len(train_dataloader) * num_epochswarmup_steps = int(total_steps * 0.1)  # 10% 的预热步数# 学习率调度器scheduler = get_linear_schedule_with_warmup(optimizer,num_warmup_steps=warmup_steps,num_training_steps=total_steps)# 混合精度训练的梯度缩放器scaler = GradScaler()# 创建TensorBoard写入器writer = SummaryWriter(log_dir='./runs/translation_experiment')

import torchimport numpy as npfrom tqdm import tqdmfrom typing import Dict, Listdef evaluate(model, dataloader, device, max_samples=None):model.eval()predictions = []references = []total_loss = 0  # 这里用于累积验证集损失num_samples = 0dataloader_iter = iter(dataloader)with torch.no_grad():for _ in tqdm(range(len(dataloader)), desc='Evaluating'):try:batch = next(dataloader_iter)except StopIteration:break# 准备输入input_ids = batch['input_ids'].to(device)attention_mask = batch['attention_mask'].to(device)labels = batch['labels'].to(device)# 计算损失（和训练集使用相同的方式）outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)loss = outputs.losstotal_loss += loss.item() * labels.size(0)  # 累积批次损失# 生成翻译结果generated_ids = model.generate(input_ids=input_ids,attention_mask=attention_mask,max_length=128,  # 减少最大长度num_beams=4,    # 降低beam sizeearly_stopping=True)# 解码预测和参考句子batch_predictions = [tokenizer.decode(g, skip_special_tokens=True) for g in generated_ids]batch_references = [tokenizer.decode(l, skip_special_tokens=True) for l in labels]predictions.extend(batch_predictions)references.extend(batch_references)# 限制评估样本数量num_samples += len(batch_predictions)if max_samples is not None and num_samples >= max_samples:break# 计算平均验证集损失avg_loss = total_loss / num_samples if num_samples > 0 else 0# 计算困惑度（这里困惑度计算逻辑保持不变，基于之前的方式）# 计算总token数（和之前计算困惑度时的逻辑一致）total_tokens = sum([len(tokenizer.encode(ref)) for ref in references])perplexity = np.exp(total_loss / total_tokens) if total_tokens > 0 else 0# 计算其他评估指标metrics = {}try:# BLEU分数 (sacrebleu标准格式)from sacrebleu import corpus_bleubleu = corpus_bleu(predictions, [references])  # sacrebleu需要参考译文列表的列表metrics['BLEU'] = bleu.scoreprint(bleu.format())  # 打印标准BLEU格式输出except ImportError:print("sacrebleu未安装，无法计算标准BLEU分数。")try:# chrF分数 (sacrebleu标准格式)from sacrebleu import corpus_chrfchrf = corpus_chrf(predictions, [references])metrics['chrF'] = chrf.scoreprint(chrf.format())  # 打印标准chrF格式输出except ImportError:print("sacrebleu未安装，无法计算标准chrF分数。")try:# TER分数 (sacrebleu)from sacrebleu import corpus_terter = corpus_ter(predictions, [references])metrics['TER'] = ter.scoreprint(ter.format())  # 打印标准TER格式输出except ImportError:print("sacrebleu未安装，无法计算TER分数。")try:# ROUGE分数 (rouge-score)from rouge_score import rouge_scorerscorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)# 计算每个样本的rouge分数并取平均值rouge_scores = {'rouge1': [], 'rouge2': [], 'rougeL': []}for pred, ref in zip(predictions, references):scores = scorer.score(ref, pred)for key in rouge_scores:rouge_scores[key].append(scores[key].fmeasure)# 计算平均ROUGE分数for key in rouge_scores:metrics[key.upper()] = np.mean(rouge_scores[key])print(f"ROUGE-{key.upper()}: {metrics[key.upper()]:.4f}")except ImportError:print("rouge-score未安装，无法计算ROUGE分数。")# 打印困惑度print(f"\n困惑度 (Perplexity): {perplexity:.4f}")# 将平均验证集损失添加到指标字典中metrics['validation_loss'] = avg_lossreturn {'perplexity': perplexity,**metrics}