Python资源：python入门代码-入门代码

import codecs import logging import multiprocessing import os.path import sys import gensim import jieba from gensim.corpora import WikiCorpus from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence import warnings warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim') # 忽略警告 logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s', datefmt='%a, %d %b %Y %H:%M:%S', filename='myapp.log', filemode='w') logging.root.setLevel(level=logging.INFO ) """ Generic word2vec interface.use Wikimedia train skip-gram word2vec """ def formatXmlWikiData(filepath): # 将xml的wiki数据转换为text格式 program = os.path.basename(filepath) # 得到文件名 logger = logging.getLogger(program) logger.info ("running %s" % ' '.join(sys.argv)) if len(sys.argv) < 3: print(globals()['__doc__'] % locals()) sys.exit(1) inp, outp = sys.argv[1:3] space = " " i = 0 output = open(outp, 'w') wiki = WikiCorpus(inp, lemmatize=False, dictionary=[]) # gensim里的维基百科处理类WikiCorpus for text in wiki.get_texts(): # 通过get_texts将维基里的每篇文章转换位1行text文本，并且去掉了标点符号等内容 output.write(space.join(text) + "\n") i = i + 1 if (i % 10000 == 0): logger.info ("Saved " + str(i) + " articles.") output.close() logger.info ("Finished Saved " + str(i) + " articles.") def tokenProcess(): # 调用jieba分词对数据进行分词处理 f = codecs.open('wiki.zh.simp.txt', 'r', encoding='utf8') target = codecs.open('wiki.zh.simp.seg.txt', 'w', encoding='utf8') print('open files.') lineNum = 1 line = f.readline() while line: print('---processing ', lineNum, ' article---') seg_list = jieba.cut(line, cut_all=False) line_seg = ' '.join(seg_list) target.writelines(line_seg) lineNum = lineNum + 1 line = f.readline() print('well done.') f.close() target.close() def trainWord2Vec(filepath): # 训练word2vec模型 logger = logging.getLogger(filepath) logger.info ("running %s" % ' '.join(filepath)) # inp为输入语料, outp1 为输出模型, outp2为原始c版本word2vec的vector格式的模型 fdir = '/Users/sy/Desktop/pyRoot/wiki_zh_vec/' inp = fdir + 'wiki.zh.simp.seg.txt' outp1 = fdir + 'wiki.zh.text.model' outp2 = fdir + 'wiki.zh.text.vector' # 训练skip-gram模型 model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count()) # 保存模型 model.save(outp1) model.wv.save_word2vec_format(outp2, binary=False) def testWord2Vec(): # 测试模型 fdir = '/Users/sy/Desktop/pyRoot/wiki_zh_vec/' model = gensim.models.Word2Vec.load(fdir + 'wiki.zh.text.model') word = model.most_similar(u"足球") for t in word: print(t[0], t[1]) word = model.most_similar(positive=[u'皇上', u'国王'], negative=[u'皇后']) for t in word: print(t[0], t[1]) print(model.similarity(u'书籍', u'书本')) print(model.similarity(u'逛街', u'书本')) if __name__ == '__main__': print("BEGIN")