英文文本预处理

#coding:utf-8 import warnings warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim') import gensim from gensim import corpora, models, similarities import logging from collections import defaultdict #logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO) def Similarity(): for i in range(500): doc1=[] doc2=[] wr=[] querypath='E:\\English\\PorterStemming\\News_E_'+str(i)+'.txt' f_r2 = open('E:\\English\\PorterStemming\\News_E_'+str(i)+'.txt', 'r')#读出要比较的文档 temp = f_r2.read() doc2 = temp.split('\n') for j in range(500): f_r = open('E:\\English\\PorterStemming\\News_E_' + str(j) + '.txt', 'r') temp = f_r.read() doc = temp.split('\n') doc1.append(doc) #f_r.close() dictionary=corpora.Dictionary(doc1) print(dictionary.token2id) doc2_vec=dictionary.doc2bow(doc2) #建立语料库 corpus=[dictionary.doc2bow(doc) for doc in doc1] #建立tfidf模型 tfidf=models.TfidfModel(corpus) #将整个语料库转为tfidf表示 corpus_tfidf=tfidf[corpus] #建立索引 #index = similarities.MatrixSimilarity(corpus_tfidf) index = similarities.Similarity(querypath,corpus_tfidf, len(dictionary)) # # # 相似度计算 doc2_tfidf = tfidf[doc2_vec] sims = index[doc2_tfidf] sims=list(sims) #print(sims) for item in sims: wr.append(str(item)) f_w=open('E:\English\Simailarity\Sim_Value.txt','a') f_w.write('\t'.join(wr)+'\n') Similarity()