简单的任务型机器问答对话含语料

import jieba import re import math import xlrd import time from sklearn import feature_extraction from sklearn.feature_extraction.text import CountVectorizer from sklearn.feature_extraction.text import TfidfTransformer from sklearn.feature_extraction.text import TfidfVectorizer def cosAB(listA,listB): resulttfdif=[] for i in range(len(listB)): #print(listA,'\n',listB[i]) if len(listA)!=len(listB[i]) or len(listA)<1: return False Molecular,denominatorA,denominatorB,denominator=0,1,1,0 for j in range(len(listA)): listA[j]=float(listA[j]) listB[i][j]=float(listB[i][j]) #print('A和B',listA[j],listB[i][j]) Molecular=Molecular+(listA[j]*listB[i][j]) #print(Molecular,'\n') denominatorA+=listA[j]*listA[j] denominatorB+=listB[i][j]*listB[i][j] denominator=Molecular/(math.sqrt(denominatorA)*math.sqrt(denominatorB)) #print('分子分母',Molecular,denominatorA,denominatorB) #print(denominator,'\n') resulttfdif.append(denominator) #print('cos-result:',resulttfdif) return resulttfdif ''' def coreword(path): wordlist=[] taglist=[] i=0 fo = open(path, 'r',encoding="utf-8") taglist=[] for line in fo.readlines(): r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~“”！，：＋+、。；？ 〈\s\d]+' i+=1 line=line.strip() line=line.split(' ') listline=line[0].split('\t') taglist.append(listline[0]) linestr = re.sub(r, '', listline[1]) seg_list = jieba.cut(linestr, cut_all=False) wordlist.append(" ".join(seg_list)) #seg_str=" ".join(seg_list) #print("读取的数据为：%s"%(line)) #print("文档",i,seg_str,'\n') fo.close() #for t in wordlist: #print(t,'\n') return wordlist,taglist ''' def coreword(path): wordlist=[] labellist=[] ExcelFile=xlrd.open_workbook(path) sheet=ExcelFile.sheet_by_index(0) rownum=sheet.nrows print('优化后',rownum) r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~“”！，：＋+、。；？ 〈\s\d]+' for i in range(0,rownum): row=sheet.row_values(i) wordstr= re.sub(r, '',row[0]) seg_list = jieba.cut(wordstr, cut_all=False) wordlist.append(" ".join(seg_list)) labellist.append(row[1]) #print(wordlist,'\n',labellist,'\n') return wordlist,labellist def groupwordtfdif(wordlist): count_list=CountVectorizer() count_a1ist=count_list.fit_transform(wordlist)#计算各个词语出现的次数' word=count_list.get_feature_names()#所有文档的关键词 #tg_wordcountarray=count_a1ist.toarray().tolist() #print(tg_wordcountarray)#查看词频结果 groupransformer=TfidfVectorizer() groupransformer.fit(wordlist) groupfidf=groupransformer.transform(wordlist)#将词频矩阵X统计成TF-IDF值 group_wordtfdif=groupfidf.toarray().tolist() #print('多文档TFIDF矩阵:',group_wordtfdif) return group_wordtfdif def singlewordtfdif(newtext,wordlist): groupransformer=TfidfVectorizer() groupransformer.fit(wordlist)#用X_train数据来fit singlefidf=groupransformer.transform(newtext)#得到TF-IDF矩阵 single_wordtfdif=singlefidf.toarray().tolist() #print('新文档TFIDF矩阵:',single_wordtfdif) return single_wordtfdif def dealword(str1): wordlist=[] r = '[’!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~“”！，：＋+、。；？ 〈\s\d]+' str1=str1.strip() linestr = re.sub(r, '', str1) seg_list = jieba.cut(linestr, cut_all=False) wordlist.append(" ".join(seg_list)) return wordlist def main(): listext,listtag=coreword("D:\FAQ_data.xlsx") grouplist=groupwordtfdif(listext) while True: something=input("请提问：") start=time.time() singletext=dealword(something) newlist=singlewordtfdif(singletext,listext) resultlist=cosAB(newlist[0],grouplist) maxindex=resultlist.index(max(resultlist)) print("最大值位置",maxindex) print("回答",listtag[maxindex]) end=time.time() print('用时:',end-start,'s') if __name__=='__main__': main()