ccks_2019事件抽取比赛.zip

import json from tqdm import tqdm import os import numpy as np from random import choice from itertools import groupby import pickle mode = 0 min_count = 2 char_size = 128 with open('./data/kb.pkl','rb') as f: id2kb = pickle.load(f) kb2id=pickle.load(f) id2char = pickle.load(f) char2id = pickle.load(f) with open('./data/train.pkl','rb') as f: all_alies = pickle.load(f) train_data=all_alies random_order=range(len(train_data)) np.random.permutation(random_order) train_data = [train_data[j] for i, j in enumerate(random_order) if i % 9 != 0] for i in train_data[:10]: print(i) def seq_padding(X, padding=0): L = [len(x) for x in X] ML = max(L) return np.array([ np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X ]) char_size=128 from keras.layers import * from keras.models import Model import keras.backend as K from keras.callbacks import Callback from keras.optimizers import Adam def seq_maxpool(x): """seq是[None, seq_len, s_size]的格式， mask是[None, seq_len, 1]的格式，先除去mask部分， 然后再做maxpooling。 """ seq,mask=x seq-=(1-mask)*1e10 return K.max(seq,1) x1_in = Input(shape=(None,)) # 待识别句子输入 x2_in = Input(shape=(None,)) # 实体语义表达输入 s1_in = Input(shape=(None,)) # 实体左边界（标签） s2_in = Input(shape=(None,)) # 实体右边界（标签） y_in = Input(shape=(None,)) # 实体标记 t_in = Input(shape=(1,)) # 是否有关联（标签） x1, x2, s1, s2, y, t = x1_in, x2_in, s1_in, s2_in, y_in, t_in x1_mask = Lambda(lambda x: K.cast(K.greater(K.expand_dims(x, 2), 0), 'float32'))(x1) x2_mask = Lambda(lambda x: K.cast(K.greater(K.expand_dims(x, 2), 0), 'float32'))(x2) embedding=Embedding(len(id2char)+2,char_size) x1 = embedding(x1) x1 = Dropout(0.2)(x1) x1 = Lambda(lambda x: x[0] * x[1])([x1, x1_mask]) x1 = Bidirectional(CuDNNLSTM(char_size//2, return_sequences=True))(x1) x1 = Lambda(lambda x: x[0] * x[1])([x1, x1_mask]) x1 = Bidirectional(CuDNNLSTM(char_size//2, return_sequences=True))(x1) x1 = Lambda(lambda x: x[0] * x[1])([x1, x1_mask]) h = Conv1D(char_size, 3, activation='relu', padding='same')(x1) ps1 = Dense(1, activation='sigmoid')(h) ps2 = Dense(1, activation='sigmoid')(h) #shape=(?, ?, 1) s_model=Model(x1_in,[ps1,ps2]) y=Lambda(lambda x:K.expand_dims(x,2))(y) x1=Concatenate()([x1,y]) x1=Conv1D(char_size,3,padding='same')(x1) x2 = embedding(x2) x2 = Dropout(0.2)(x2) x2 = Lambda(lambda x: x[0] * x[1])([x2, x2_mask]) x2 = Bidirectional(CuDNNLSTM(char_size//2, return_sequences=True))(x2) x2 = Lambda(lambda x: x[0] * x[1])([x2, x2_mask]) x2 = Bidirectional(CuDNNLSTM(char_size//2, return_sequences=True))(x2) x2 = Lambda(lambda x: x[0] * x[1])([x2, x2_mask]) x1=Lambda(seq_maxpool)([x1,x1_mask]) x2=Lambda(seq_maxpool)([x2,x2_mask]) x12=Multiply()([x1,x2]) x=Concatenate()([x1,x2,x12]) x = Dense(char_size, activation='relu')(x) pt = Dense(1, activation='sigmoid')(x) # shape=(?, 1) t_model = Model([x1_in, x2_in, y_in], pt) train_model = Model([x1_in, x2_in, s1_in, s2_in, y_in, t_in], [ps1, ps2, pt]) s1 = K.expand_dims(s1, 2) s2 = K.expand_dims(s2, 2) s1_loss = K.binary_crossentropy(s1, ps1) s1_loss = K.sum(s1_loss * x1_mask) / K.sum(x1_mask) s2_loss = K.binary_crossentropy(s2, ps2) s2_loss = K.sum(s2_loss * x1_mask) / K.sum(x1_mask) pt_loss = K.mean(K.binary_crossentropy(t, pt)) loss = s1_loss + s2_loss + pt_loss train_model.add_loss(loss) train_model.compile(optimizer=Adam(1e-3)) train_model.summary() class data_generator: def __init__(self, data, batch_size=128): self.data = data self.batch_size = batch_size self.steps = len(self.data) // self.batch_size if len(self.data) % self.batch_size != 0: self.steps += 1 def __len__(self): return self.steps def __iter__(self): while True: idxs = range(len(self.data)) X1, X2, S1, S2, Y, T = [], [], [], [], [], [] for i in idxs: try: text=self.data[i][0] x1 = [char2id.get(c, 1) for c in text] s1, s2 = np.array(self.data[i][-2]),np.array(self.data[i][-1]) x2 = self.data[i][4] x2 = [char2id.get(c, 1) for c in x2] X1.append(x1) X2.append(x2) S1.append(s1) S2.append(s2) Y.append(self.data[i][2]) T.append(self.data[i][3]) if len(X1) == self.batch_size or i == idxs[-1]: X1 = seq_padding(X1) X2 = seq_padding(X2) S1 = seq_padding(S1) S2 = seq_padding(S2) Y = seq_padding(Y) T = seq_padding(T) yield [X1, X2, S1, S2, Y, T], None X1, X2, S1, S2, Y, T = [], [], [], [], [], [] except: pass def find_lcseque(s1, s2): # 生成字符串长度加1的0矩阵，m用来保存对应位置匹配的结果 m = [[0 for x in range(len(s2) + 1)] for y in range(len(s1) + 1)] # d用来记录转移方向 d = [[None for x in range(len(s2) + 1)] for y in range(len(s1) + 1)] for p1 in range(len(s1)): for p2 in range(len(s2)): if s1[p1] == s2[p2]: # 字符匹配成功，则该位置的值为左上方的值加1 m[p1 + 1][p2 + 1] = m[p1][p2] + 1 d[p1 + 1][p2 + 1] = 'ok' elif m[p1 + 1][p2] > m[p1][p2 + 1]: # 左值大于上值，则该位置的值为左值，并标记回溯时的方向 m[p1 + 1][p2 + 1] = m[p1 + 1][p2] d[p1 + 1][p2 + 1] = 'left' else: # 上值大于左值，则该位置的值为上值，并标记方向up m[p1 + 1][p2 + 1] = m[p1][p2 + 1] d[p1 + 1][p2 + 1] = 'up' (p1, p2) = (len(s1), len(s2)) s = [] while m[p1][p2]: # 不为None时 c = d[p1][p2] if c == 'ok': # 匹配成功，插入该字符，并向左上角找下一个 s.append(s1[p1 - 1]) p1 -= 1 p2 -= 1 if c == 'left': # 根据标记，向左找下一个 p2 -= 1 if c == 'up': # 根据标记，向上找下一个 p1 -= 1 s.reverse() return ''.join(s) def match_id(s): match_data_dic = {} for k, v in kb2id.items(): match_length = len(find_lcseque(s, k)) match_data_dic[' '.join(v)] = match_length match_data_dic = sorted(match_data_dic.items(), key=lambda item: item[1], reverse=True) i = 0 id_list = [] for k in match_data_dic: id_list.extend(k[0].split(' ')) i += 1 if i >=4: break id_list = list(set(id_list)) return id_list def extract_items(text_in): _x1 = [char2id.get(c, 1) for c in text_in] _x1 = np.array([_x1]) _k1, _k2 = s_model.predict(_x1) _k1, _k2 = _k1[0, :, 0], _k2[0, :, 0] _k1, _k2 = np.where(_k1 > 0.5)[0], np.where(_k2 > 0.5)[0] _subjects = [] for i in _k1: j = _k2[_k2 >= i] if len(j) > 0: j = j[0] _subject = text_in[i: j+1] _subjects.append((_subject, i, j)) print(_subjects) if _subjects: R = [] _X2, _Y = [], [] _S, _IDXS = [], {} for _s in _subjects: _y = np.zeros(len(text_in)) _y[_s[1]: _s[2]] = 1 _IDXS[_s] = kb2id.get(_s[0], match_id(_s[0])) for i in _IDXS[_s]: _x2 = id2kb[i]['subject_desc'] _x2 = [char2id.get(c, 1) for c in _x2] _X2.append(_x2) _Y.append(_y) _S.append(_s) if _X2: _X2 = seq_padding(_X2) _Y = seq_padding(_Y) _X1 = np.repeat(_x