Basic-DCN-Demo.zip

import numpy as np import tensorflow as tf from time import time from sklearn.base import BaseEstimator, TransformerMixin from sklearn.metrics import roc_auc_score class DCN(BaseEstimator, TransformerMixin): def __init__(self, cate_feature_size, field_size,numeric_feature_size, embedding_size=8, deep_layers=[32, 32], dropout_deep=[0.5, 0.5, 0.5], deep_layers_activation=tf.nn.relu, epoch=10, batch_size=256, learning_rate=0.001, optimizer_type="adam", batch_norm=0, batch_norm_decay=0.995, verbose=False, random_seed=2016, loss_type="logloss", eval_metric=roc_auc_score, l2_reg=0.0, greater_is_better=True,cross_layer_num=3): assert loss_type in ["logloss", "mse"], \ "loss_type can be either 'logloss' for classification task or 'mse' for regression task" self.cate_feature_size = cate_feature_size self.numeric_feature_size = numeric_feature_size self.field_size = field_size self.embedding_size = embedding_size self.total_size = self.field_size * self.embedding_size + self.numeric_feature_size self.deep_layers = deep_layers self.cross_layer_num = cross_layer_num self.dropout_dep = dropout_deep self.deep_layers_activation = deep_layers_activation self.l2_reg = l2_reg self.epoch = epoch self.batch_size = batch_size self.learning_rate = learning_rate self.optimizer_type = optimizer_type self.batch_norm = batch_norm self.batch_norm_decay = batch_norm_decay self.verbose = verbose self.random_seed = random_seed self.loss_type = loss_type self.eval_metric = eval_metric self.greater_is_better = greater_is_better self.train_result,self.valid_result = [],[] self._init_graph() def _init_graph(self): self.graph = tf.Graph() with self.graph.as_default(): tf.set_random_seed(self.random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None,None], name='feat_index') self.feat_value = tf.placeholder(tf.float32, shape=[None,None], name='feat_value') self.numeric_value = tf.placeholder(tf.float32,[None,None],name='num_value') self.label = tf.placeholder(tf.float32,shape=[None,1],name='label') self.dropout_keep_deep = tf.placeholder(tf.float32,shape=[None],name='dropout_deep_deep') self.train_phase = tf.placeholder(tf.bool,name='train_phase') self.weights = self._initialize_weights() # model self.embeddings = tf.nn.embedding_lookup(self.weights['feature_embeddings'],self.feat_index) # N * F * K feat_value = tf.reshape(self.feat_value,shape=[-1,self.field_size,1]) self.embeddings = tf.multiply(self.embeddings,feat_value) self.x0 = tf.concat([self.numeric_value, tf.reshape(self.embeddings,shape=[-1,self.field_size * self.embedding_size])] ,axis=1) # deep part self.y_deep = tf.nn.dropout(self.x0,self.dropout_keep_deep[0]) for i in range(0,len(self.deep_layers)): self.y_deep = tf.add(tf.matmul(self.y_deep,self.weights["deep_layer_%d" %i]), self.weights["deep_bias_%d"%i]) self.y_deep = self.deep_layers_activation(self.y_deep) self.y_deep = tf.nn.dropout(self.y_deep,self.dropout_keep_deep[i+1]) # cross_part self._x0 = tf.reshape(self.x0, (-1, self.total_size, 1)) x_l = self._x0 for l in range(self.cross_layer_num): x_l = tf.tensordot(tf.matmul(self._x0, x_l, transpose_b=True), self.weights["cross_layer_%d" % l],1) + self.weights["cross_bias_%d" % l] + x_l self.cross_network_out = tf.reshape(x_l, (-1, self.total_size)) # concat_part concat_input = tf.concat([self.cross_network_out, self.y_deep], axis=1) self.out = tf.add(tf.matmul(concat_input,self.weights['concat_projection']),self.weights['concat_bias']) # loss if self.loss_type == "logloss": self.out = tf.nn.sigmoid(self.out) self.loss = tf.losses.log_loss(self.label, self.out) elif self.loss_type == "mse": self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out)) # l2 regularization on weights if self.l2_reg > 0: self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["concat_projection"]) for i in range(len(self.deep_layers)): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["deep_layer_%d" % i]) for i in range(self.cross_layer_num): self.loss += tf.contrib.layers.l2_regularizer( self.l2_reg)(self.weights["cross_layer_%d" % i]) if self.optimizer_type == "adam": self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8).minimize(self.loss) elif self.optimizer_type == "adagrad": self.optimizer = tf.train.AdagradOptimizer(learning_rate=self.learning_rate, initial_accumulator_value=1e-8).minimize(self.loss) elif self.optimizer_type == "gd": self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate).minimize(self.loss) elif self.optimizer_type == "momentum": self.optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.95).minimize( self.loss) #init self.saver = tf.train.Saver() init = tf.global_variables_initializer() self.sess = tf.Session() self.sess.run(init) # number of params total_parameters = 0 for variable in self.weights.values(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters if self.verbose > 0: print("#params: %d" % total_parameters) def _initialize_weights(self): weights = dict() #embeddings weights['feature_embeddings'] = tf.Variable( tf.random_normal([self.cate_feature_size,self.embedding_size],0.0,0.01), name='feature_embeddings') weights['feature_bias'] = tf.Variable(tf.random_normal([self.cate_feature_size,1],0.0,1.0),name='feature_bias') #deep layers num_layer = len(self.deep_layers) glorot = np.sqrt(2.0/(self.total_size + self.deep_layers[0])) weights['deep_layer_0'] = tf.Variable( np.random.normal(loc=0,scale=glorot,size=(self.total_size,self.deep_layers[0])),dtype=np.float32 ) weights['deep_bias_0'] = tf.Variable( np.random.normal(loc=0,scale=glorot,size=(1,self.deep_layers[0])),dtype=np.float32 ) for i in range(1,num_layer): glorot = np.sqrt(2.0 / (self.deep_layers[i - 1] + self.deep_layers[i])) weights["deep_layer_%d" % i] = tf.Variable( np.random.normal(loc=0, scale=glorot, size=(self.deep_layers[i - 1], self.deep_layers[i])),