基于tensorflow深度学习的中文机器阅读理解-完形填空.zip

""" """ from __future__ import print_function import tensorflow as tf import numpy as np import time import os class Config(object): """RNN配置参数""" file_name = 'rnn' # 保存模型文件 use_embedding = True # 是否用词向量，否则one_hot embedding_dim = 60 # 词向量维度 num_layers= 1 # 隐藏层层数 hidden_dim = 64 # 隐藏层神经元 train_keep_prob = 0.8 # dropout保留比例 learning_rate = 1e-3 # 学习率 batch_size = 60 # 每批训练大小 max_steps = 20000 # 总迭代batch数 log_every_n = 10 # 每多少轮输出一次结果 save_every_n = 50 # 每多少轮校验模型并保存 class Model(object): def __init__(self, config, vocab_size, embedding_array): self.config = config self.vocab_size = vocab_size self.embedding = embedding_array # 待输入的数据 self.query_seqs = tf.placeholder(tf.int32, [None, None], name='query') self.query_length = tf.placeholder(tf.int32, [None], name='query_length') self.response_seqs = tf.placeholder(tf.int32, [None, None], name='response') self.response_length = tf.placeholder(tf.int32, [None], name='response_length') self.targets = tf.placeholder(tf.float32, shape=[None, None], name='targets') self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') # 两个全局变量 self.global_step = tf.Variable(0, trainable=False, name="global_step") self.global_loss = tf.Variable(0, dtype=tf.float32, trainable=False, name="global_loss") # Ann模型 self.rnn() # 初始化session self.saver = tf.train.Saver() self.session = tf.Session() self.session.run(tf.global_variables_initializer()) def rnn(self): """rnn模型""" def get_mul_cell(hidden_dim, num_layers):# 创建多层lstm def get_en_cell(hidden_dim):# 创建单个lstm enc_base_cell = tf.nn.rnn_cell.BasicLSTMCell(hidden_dim, forget_bias=1.0) return enc_base_cell return tf.nn.rnn_cell.MultiRNNCell([get_en_cell(hidden_dim) for _ in range(num_layers)]) def bilstm(hidm, seq, seq_len, num_layers=1): cell_fw = get_mul_cell(hidm, num_layers) cell_bw = get_mul_cell(hidm, num_layers) (output_fw, output_bw), state = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, seq,sequence_length=seq_len,dtype=tf.float32) output = tf.concat([output_fw, output_bw], axis=-1) return output, state # 词嵌入层 if self.config.use_embedding is False: self.lstm_query_seqs = tf.one_hot(self.query_seqs,depth=self.vocab_size) # 独热编码[1,2,3] depth=5 --> [[0,1,0,0,0],[0,0,1,0,0],[0,0,0,1,0]]，此时的输入节点个数为num_classes self.lstm_response_seqs = tf.one_hot(self.response_seqs, depth=self.vocab_size) else: embedding = tf.get_variable('embedding', [self.vocab_size, self.config.embedding_dim]) # 可训练的词向量 # embedding = tf.constant(self.embedding, dtype=tf.float32) # 用自定义词向量 embedding_zero = tf.constant(0, dtype=tf.float32, shape=[1, self.config.embedding_dim]) embedding = tf.concat([embedding, embedding_zero], axis=0) # 增加一行0向量，代表padding向量值 self.lstm_query_seqs = tf.nn.embedding_lookup(embedding,self.query_seqs) # 词嵌入[1,2,3] --> [[3,...,4],[0.7,...,-3],[6,...,9]],embeding[depth*embedding_size]=[[0.2,...,6],[3,...,4],[0.7,...,-3],[6,...,9],[8,...,-0.7]]，此时的输入节点个数为embedding_size self.lstm_response_seqs = tf.nn.embedding_lookup(embedding, self.response_seqs) self.lstm_query_seqs = tf.nn.dropout(self.lstm_query_seqs, keep_prob=self.keep_prob) self.lstm_response_seqs = tf.nn.dropout(self.lstm_response_seqs, keep_prob=self.keep_prob) with tf.variable_scope("lstm_layer1") as scope: # 第一层bilstm网络 query_output, self.query_state = bilstm(self.config.hidden_dim, self.lstm_query_seqs, self.query_length, self.config.num_layers) scope.reuse_variables() # bilstm共享 response_output, self.response_state = bilstm(self.config.hidden_dim, self.lstm_response_seqs, self.response_length, self.config.num_layers) with tf.variable_scope("lstm_layer2") as scope: # 第二层bilstm网络，第一层的输出作为第二层的输入 query_output, self.query_state = bilstm(self.config.hidden_dim, query_output, self.query_length, self.config.num_layers) scope.reuse_variables() # bilstm共享，但和第一层lstm_layer1不共享。 response_output, self.response_state = bilstm(self.config.hidden_dim, response_output, self.response_length, self.config.num_layers) query_c_fw, query_h_fw = self.query_state[0][-1] # 前向最后一层c/h query_c_bw, query_h_bw = self.query_state[1][-1] # 后向最后一层c/h response_c_fw, response_h_fw = self.response_state[0][-1] response_c_bw, response_h_bw = self.response_state[1][-1] self.query_h_state = tf.concat([query_h_fw, query_h_bw], axis=1) self.response_h_state = tf.concat([response_h_fw, response_h_bw], axis=1) outputs = tf.concat([query_output, response_output], 1) final_state = (self.query_h_state + self.response_h_state) / 2 # 双向拼接、上下文取平均，得到encode向量 L_context_mask = (1 - tf.cast(tf.sequence_mask(self.query_length), tf.float32)) * ( -1e12) # 对填充位置进行mask，注意这里是softmax之前的mask，所以mask不是乘以0，而是减去1e12 R_context_mask = (1 - tf.cast(tf.sequence_mask(self.response_length), tf.float32)) * (-1e12) context_mask = tf.concat([L_context_mask, R_context_mask], 1) attention = context_mask + tf.matmul(outputs, tf.expand_dims(final_state, 2))[:, :,0] # encode向量与每个时间步状态向量做内积，然后mask，然后softmax with tf.name_scope("optimize"): # 损失函数，交叉熵 self.losses = tf.nn.softmax_cross_entropy_with_logits(labels=self.targets, logits=attention) self.mean_loss = tf.reduce_mean(self.losses, name="mean_loss") # batch样本的平均损失 # 优化器 self.optim = tf.train.AdamOptimizer(learning_rate=self.config.learning_rate).minimize(self.mean_loss, global_step=self.global_step) with tf.name_scope("score"): self.y_pre = tf.nn.softmax(attention) # self.y_cos = self.logits # self.y_pre = tf.to_int32((tf.sign(self.y_cos * 2 - 1) + 1) / 2) def load(self, checkpoint): self.saver.restore(self.session, checkpoint) print('Restored from: {}'.format(checkpoint)) def train(self, batch_train_g, model_path, val_g): with self.session as sess: for q, q_len, r, r_len, y in batch_train_g: start = time.time() feed = {self.query_seqs: q, self.query_length: q_len, self.response_seqs: r, self.response_length: r_len, self.targets: y, self.keep_prob: self.config.train_keep_prob} batch_loss, _, query_h_state,lstm_query_seqs= sess.run([self.mean_loss, self.optim,self.query_h_state,self.lstm_query_seqs],feed_dict=feed) end = time.time() # control the print lines if self.global_step.eval() % self.config.log_every_n == 0: print('step: {}/{}... '.format(self.global_step.eval(), self.con