Hugging Face-环境配置

打开anaconda prompt

conda activate pytorch

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple transformers datasets tokenizers

pycharm

 找到pytorch下的python.exe

#将模型下载到本地调用
from transformers import AutoModelForCausalLM,AutoTokenizer#将模型和分词工具下载到本地，并指定保存路径
model_name = "uer/gpt2-chinese-cluecorpussmall"
cache_dir = "model/uer/gpt2-chinese-cluecorpussmall"#下载模型
AutoModelForCausalLM.from_pretrained(model_name, cache_dir=cache_dir)
#下载分词工具
AutoTokenizer.from_pretrained(model_name,cache_dir=cache_dir)print(f"模型分词器已下载到：{cache_dir}")

from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline# 设置具体包含config.json的目录
model_dir = r"D:\pycharm\sth\demo\transFormers_test\model\uer\gpt2-chinese-cluecorpussmall\models--uer--gpt2-chinese-cluecorpussmall\snapshots\c2c0249d8a2731f269414cc3b22dff021f8e07a3"# 将模型和分词工具下载到本地，并指定保存路径
model = AutoModelForCausalLM.from_pretrained(model_dir)
tokenizer = AutoTokenizer.from_pretrained(model_dir)# 使用模型和分词器创建生成文本的pipeline
generator = pipeline("text-generation", model=model, tokenizer=tokenizer,device="cuda")# 生成内容
# output = generator("你好，我是一款语言模型，",max_length=50,num_return_sequences=1)output = generator("你好，我是一款语言模型，",num_return_sequences=1, # 设置返回多少个独立的生成序列max_length=50,truncation=True,# 生成文本以适应文本最大长度temperature=0.7,# 控制文本生成的随机性，值越高，生成多样性越好top_k=50, # 限制模型在每一步生成时仅从概率最高的k个词中随机选择下一个词top_p=0.9, # 进一步限制模型生成时的词汇选择范围，选择一组概率累计达到p的词汇，模型只会从这个概率集合中采样clean_up_tokenization_spaces=True # 设置生成文本分词时的空格是否保留
)
print(output)

from datasets import load_dataset, load_from_disk# # 在线加载
dataset = load_dataset("lansinuote/ChnSentiCorp", cache_dir="E:/DeepLearning/data")
# # 保存为可本地加载的格式
save_path = "E:/DeepLearning/data/my_chn_senti_corp"
# dataset.save_to_disk(save_path)
# 从本地加载
loaded_dataset = load_from_disk(save_path)
print(loaded_dataset)test_data = dataset["test"]
print(test_data)
for data in test_data:print(data)