pull dev for v0.4.1

add DSIN
refactor layers

README.md

@@ -36,8 +36,9 @@ Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Star
 |         Deep Interest Network          | [KDD 2018][Deep Interest Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1706.06978.pdf)                                                       |
 |    Deep Interest Evolution Network     | [AAAI 2019][Deep Interest Evolution Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1809.03672.pdf)                                           |
 |                AutoInt                 | [arxiv 2018][AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921)                              |
-|                  NFFM                  | [arxiv 2019][Field-aware Neural Factorization Machine for Click-Through Rate Prediction ](https://arxiv.org/pdf/1902.09096.pdf) (The original NFFM was first used by Yi Yang([email protected]) in TSA competition in 2017.)                                 |
-|                  FGCNN                  | [WWW 2019][Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction ](https://arxiv.org/pdf/1904.04447))   
+|                  NFFM                  | [arxiv 2019][Field-aware Neural Factorization Machine for Click-Through Rate Prediction ](https://arxiv.org/pdf/1902.09096.pdf) (The original NFFM was used by Yi Yang([email protected]) in TSA competition.)                                 |
+|                  FGCNN                  | [WWW 2019][Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction ](https://arxiv.org/pdf/1904.04447)   |
+|                  Deep Session Interest Network                | [IJCAI 2019][Deep Session Interest Network for Click-Through Rate Prediction ](https://arxiv.org/abs/1905.06482)  |
 
 
 

deepctr/__init__.py

@@ -2,5 +2,5 @@
 from . import models
 from .utils import check_version
 
-__version__ = '0.4.0'
+__version__ = '0.4.1'
 check_version(__version__)

deepctr/input_embedding.py

@@ -13,62 +13,60 @@
 from tensorflow.python.keras.layers import Concatenate, Dense, Embedding, Input, Reshape, add
 from tensorflow.python.keras.regularizers import l2
 
-from deepctr.layers import Hash
 from .layers.sequence import SequencePoolingLayer
 from .layers.utils import Hash
 
 
 def create_singlefeat_inputdict(feature_dim_dict, prefix=''):
     sparse_input = OrderedDict()
-    for i, feat in enumerate(feature_dim_dict["sparse"]):
+    for feat in feature_dim_dict["sparse"]:
         sparse_input[feat.name] = Input(
-            shape=(1,), name=feat.name, dtype=feat.dtype)  # prefix+'sparse_' + str(i) + '-' + feat.name)
+            shape=(1,), name=prefix+feat.name, dtype=feat.dtype)
 
     dense_input = OrderedDict()
 
-    for i, feat in enumerate(feature_dim_dict["dense"]):
+    for  feat in feature_dim_dict["dense"]:
         dense_input[feat.name] = Input(
-            shape=(1,), name=feat.name)  # prefix+'dense_' + str(i) + '-' + feat.name)
+            shape=(1,), name=prefix+feat.name,dtype=feat.dtype)
 
     return sparse_input, dense_input
 
 
-def create_varlenfeat_inputdict(feature_dim_dict, mask_zero=True):
+def create_varlenfeat_inputdict(feature_dim_dict, mask_zero=True,prefix=''):
     sequence_dim_dict = feature_dim_dict.get('sequence', [])
     sequence_input_dict = OrderedDict()
-    for i, feat in enumerate(sequence_dim_dict):
-        sequence_input_dict[feat.name] = Input(shape=(feat.maxlen,), name='seq_' + str(
-            i) + '-' + feat.name, dtype=feat.dtype)
+    for feat in sequence_dim_dict:
+        sequence_input_dict[feat.name] = Input(shape=(feat.maxlen,), name=prefix+'seq_' + feat.name, dtype=feat.dtype)
 
     if mask_zero:
         sequence_len_dict, sequence_max_len_dict = None, None
     else:
         sequence_len_dict = {feat.name: Input(shape=(
-            1,), name='seq_length' + str(i) + '-' + feat.name) for i, feat in enumerate(sequence_dim_dict)}
+            1,), name=prefix+'seq_length_' + feat.name) for  feat in sequence_dim_dict}
         sequence_max_len_dict = {feat.name: feat.maxlen
-                                 for i, feat in enumerate(sequence_dim_dict)}
+                                 for  feat in sequence_dim_dict}
     return sequence_input_dict, sequence_len_dict, sequence_max_len_dict
 
 
-def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_reg, prefix='sparse',
+def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_reg, prefix='sparse_',
                           seq_mask_zero=True):
     if embedding_size == 'auto':
         print("Notice:Do not use auto embedding in models other than DCN")
         sparse_embedding = {feat.name: Embedding(feat.dimension, 6 * int(pow(feat.dimension, 0.25)),
                                                  embeddings_initializer=RandomNormal(
                                                      mean=0.0, stddev=init_std, seed=seed),
                                                  embeddings_regularizer=l2(l2_reg),
-                                                 name=prefix + '_emb_' + str(i) + '-' + feat.name) for i, feat in
-                            enumerate(feature_dim_dict["sparse"])}
+                                                 name=prefix + 'emb_' + feat.name) for feat in
+                                    feature_dim_dict["sparse"]}
     else:
 
         sparse_embedding = {feat.name: Embedding(feat.dimension, embedding_size,
                                                  embeddings_initializer=RandomNormal(
                                                      mean=0.0, stddev=init_std, seed=seed),
                                                  embeddings_regularizer=l2(
                                                      l2_reg),
-                                                 name=prefix + '_emb_' + str(i) + '-' + feat.name) for i, feat in
-                            enumerate(feature_dim_dict["sparse"])}
+                                                 name=prefix + 'emb_'  + feat.name) for feat in
+                                    feature_dim_dict["sparse"]}
 
     if 'sequence' in feature_dim_dict:
         count = len(sparse_embedding)
@@ -81,7 +79,7 @@ def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_r
                                                             mean=0.0, stddev=init_std, seed=seed),
                                                         embeddings_regularizer=l2(
                                                             l2_reg),
-                                                        name=prefix + '_emb_' + str(count) + '-' + feat.name,
+                                                        name=prefix + 'seq_emb_' + feat.name,
                                                         mask_zero=seq_mask_zero)
 
             else:
@@ -90,7 +88,7 @@ def create_embedding_dict(feature_dim_dict, embedding_size, init_std, seed, l2_r
                                                             mean=0.0, stddev=init_std, seed=seed),
                                                         embeddings_regularizer=l2(
                                                             l2_reg),
-                                                        name=prefix + '_emb_' + str(count) + '-' + feat.name,
+                                                        name=prefix + 'seq_emb_' + feat.name,
                                                         mask_zero=seq_mask_zero)
 
             count += 1

deepctr/layers/__init__.py

@@ -7,7 +7,7 @@
                           OutterProductLayer, FGCNNLayer)
 from .normalization import LayerNormalization
 from .sequence import (AttentionSequencePoolingLayer, BiasEncoding, BiLSTM,
-                       KMaxPooling, Position_Embedding, SequencePoolingLayer,
+                       KMaxPooling, SequencePoolingLayer,
                        Transformer, DynamicGRU)
 from .utils import NoMask, Hash
 

deepctr/layers/activation.py

@@ -59,14 +59,14 @@ def get_config(self, ):
         base_config = super(Dice, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
-def activation_fun(activation, fc):
+def activation_layer(activation):
     if activation == "dice" or activation == "Dice":
-        fc = Dice()(fc)
+        act_layer =  Dice()
     elif (isinstance(activation, str)) or (sys.version_info.major == 2 and isinstance(activation, (str, unicode))):
-        fc = tf.keras.layers.Activation(activation)(fc)
+        act_layer = tf.keras.layers.Activation(activation)
     elif issubclass(activation, Layer):
-        fc = activation()(fc)
+        act_layer = activation()
     else:
         raise ValueError(
             "Invalid activation,found %s.You should use a str or a Activation Layer Class." % (activation))
-    return fc
+    return act_layer

deepctr/layers/core.py

@@ -12,7 +12,7 @@
 from tensorflow.python.keras.layers import Layer
 from tensorflow.python.keras.regularizers import l2
 
-from .activation import activation_fun
+from .activation import activation_layer
 
 
 class LocalActivationUnit(Layer):
@@ -76,8 +76,12 @@ def build(self, input_shape):
                                       name="kernel")
         self.bias = self.add_weight(
             shape=(1,), initializer=Zeros(), name="bias")
-        #self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg,
-        #               self.dropout_rate, self.use_bn, seed=self.seed)
+        self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg,
+                      self.dropout_rate, self.use_bn, seed=self.seed)
+
+        self.dense = tf.keras.layers.Lambda(lambda x:tf.nn.bias_add(tf.tensordot(
+            x[0], x[1], axes=(-1, 0)), x[2]))
+
         super(LocalActivationUnit, self).build(
             input_shape)  # Be sure to call this somewhere!
 
@@ -91,10 +95,9 @@ def call(self, inputs, training=None, **kwargs):
         att_input = tf.concat(
             [queries, keys, queries - keys, queries * keys], axis=-1)
 
-        att_out = DNN(self.hidden_units, self.activation, self.l2_reg,
-                       self.dropout_rate, self.use_bn, seed=self.seed)(att_input, training=training)
-        attention_score = tf.keras.layers.Lambda(lambda x:tf.nn.bias_add(tf.tensordot(
-            x[0], x[1], axes=(-1, 0)), x[2]))([att_out,self.kernel,self.bias])
+        att_out = self.dnn(att_input, training=training)
+
+        attention_score = self.dense([att_out,self.kernel,self.bias])
 
         return attention_score
 
@@ -157,6 +160,12 @@ def build(self, input_shape):
                                      shape=(self.hidden_units[i],),
                                      initializer=Zeros(),
                                      trainable=True) for i in range(len(self.hidden_units))]
+        if self.use_bn:
+            self.bn_layers = [tf.keras.layers.BatchNormalization() for _ in range(len(self.hidden_units))]
+
+        self.dropout_layers = [tf.keras.layers.Dropout(self.dropout_rate,seed=self.seed+i) for i in range(len(self.hidden_units))]
+
+        self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))]
 
         super(DNN, self).build(input_shape)  # Be sure to call this somewhere!
 
@@ -171,10 +180,11 @@ def call(self, inputs, training=None, **kwargs):
             #           kernel_initializer=glorot_normal(seed=self.seed), \
             #           kernel_regularizer=l2(self.l2_reg))(deep_input)
             if self.use_bn:
-                fc = tf.keras.layers.BatchNormalization()(fc, training=training)
-            fc = activation_fun(self.activation, fc)
+                fc = self.bn_layers[i](fc, training=training)
+
+            fc = self.activation_layers[i](fc)
 
-            fc = tf.keras.layers.Dropout(self.dropout_rate,seed=self.seed)(fc, training=training)
+            fc = self.dropout_layers[i](fc,training = training)
             deep_input = fc
 
         return deep_input

deepctr/layers/interaction.py

@@ -14,8 +14,9 @@
                                                   glorot_uniform)
 from tensorflow.python.keras.layers import Layer
 from tensorflow.python.keras.regularizers import l2
+from tensorflow.python.layers import utils
 
-from .activation import activation_fun
+from .activation import activation_layer
 from .utils import concat_fun
 
 
@@ -87,6 +88,8 @@ def build(self, input_shape):
             embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p")
         self.dropout = tf.keras.layers.Dropout(self.dropout_rate, seed=self.seed)
 
+        self.tensordot = tf.keras.layers.Lambda(lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))
+
         # Be sure to call this somewhere!
         super(AFMLayer, self).build(input_shape)
 
@@ -119,9 +122,7 @@ def call(self, inputs, training=None, **kwargs):
 
         attention_output = self.dropout(attention_output)  # training
 
-        afm_out = tf.keras.layers.Lambda(lambda x: tf.tensordot(x[0], x[1]
-                                                                , axes=(-1, 0)))([attention_output, self.projection_p])
-
+        afm_out = self.tensordot([attention_output, self.projection_p])
         return afm_out
 
     def compute_output_shape(self, input_shape):
@@ -246,6 +247,8 @@ def build(self, input_shape):
             else:
                 self.field_nums.append(size)
 
+        self.activation_layers = [activation_layer(self.activation) for _ in self.layer_size]
+
         super(CIN, self).build(input_shape)  # Be sure to call this somewhere!
 
     def call(self, inputs, **kwargs):
@@ -275,7 +278,7 @@ def call(self, inputs, **kwargs):
 
             curr_out = tf.nn.bias_add(curr_out, self.bias[idx])
 
-            curr_out = activation_fun(self.activation, curr_out)
+            curr_out = self.activation_layers[idx](curr_out)
 
             curr_out = tf.transpose(curr_out, perm=[0, 2, 1])
 
@@ -783,6 +786,26 @@ def build(self, input_shape):
         if len(input_shape) != 3:
             raise ValueError(
                 "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape)))
+        self.conv_layers = []
+        self.pooling_layers = []
+        self.dense_layers = []
+        pooling_shape = input_shape.as_list() + [1, ]
+        embedding_size = input_shape[-1].value
+        for i in range(1, len(self.filters) + 1):
+            filters = self.filters[i - 1]
+            width = self.kernel_width[i - 1]
+            new_filters = self.new_maps[i - 1]
+            pooling_width = self.pooling_width[i - 1]
+            conv_output_shape = self._conv_output_shape(pooling_shape, (width, 1))
+            pooling_shape = self._pooling_output_shape(conv_output_shape, (pooling_width, 1))
+            self.conv_layers.append(tf.keras.layers.Conv2D(filters=filters, kernel_size=(width, 1), strides=(1, 1),
+                                                           padding='same',
+                                                           activation='tanh', use_bias=True, ))
+            self.pooling_layers.append(tf.keras.layers.MaxPooling2D(pool_size=(pooling_width, 1)))
+            self.dense_layers.append(tf.keras.layers.Dense(pooling_shape[1] * embedding_size * new_filters,
+                                                           activation='tanh', use_bias=True))
+
+        self.flatten = tf.keras.layers.Flatten()
 
         super(FGCNNLayer, self).build(
             input_shape)  # Be sure to call this somewhere!
@@ -794,24 +817,24 @@ def call(self, inputs, **kwargs):
                 "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
 
         embedding_size = inputs.shape[-1].value
-        pooling_result = tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=3))(inputs)
+        pooling_result = tf.expand_dims(inputs, axis=3)
 
         new_feature_list = []
 
         for i in range(1, len(self.filters) + 1):
-            filters = self.filters[i - 1]
-            width = self.kernel_width[i - 1]
             new_filters = self.new_maps[i - 1]
-            pooling_width = self.pooling_width[i - 1]
-            conv_result = tf.keras.layers.Conv2D(filters=filters, kernel_size=(width, 1), strides=(1, 1),
-                                                 padding='same',
-                                                 activation='tanh', use_bias=True, )(pooling_result)
-            pooling_result = tf.keras.layers.MaxPooling2D(pool_size=(pooling_width, 1))(conv_result)
-            flatten_result = tf.keras.layers.Flatten()(pooling_result)
-            new_result = tf.keras.layers.Dense(pooling_result.shape[1].value * embedding_size * new_filters,
-                                               activation='tanh', use_bias=True)(flatten_result)
+
+            conv_result = self.conv_layers[i - 1](pooling_result)
+
+            pooling_result = self.pooling_layers[i - 1](conv_result)
+
+            flatten_result = self.flatten(pooling_result)
+
+            new_result = self.dense_layers[i - 1](flatten_result)
+
             new_feature_list.append(
-                tf.keras.layers.Reshape((pooling_result.shape[1].value * new_filters, embedding_size))(new_result))
+                tf.reshape(new_result, (-1, pooling_result.shape[1].value * new_filters, embedding_size)))
+
         new_features = concat_fun(new_feature_list, axis=1)
         return new_features
 
@@ -832,3 +855,28 @@ def get_config(self, ):
                   'pooling_width': self.pooling_width}
         base_config = super(FGCNNLayer, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
+
+    def _conv_output_shape(self, input_shape, kernel_size):
+        # channels_last
+        space = input_shape[1:-1]
+        new_space = []
+        for i in range(len(space)):
+            new_dim = utils.conv_output_length(
+                space[i],
+                kernel_size[i],
+                padding='same',
+                stride=1,
+                dilation=1)
+            new_space.append(new_dim)
+        return ([input_shape[0]] + new_space + [self.filters])
+
+    def _pooling_output_shape(self, input_shape, pool_size):
+        # channels_last
+
+        rows = input_shape[1]
+        cols = input_shape[2]
+        rows = utils.conv_output_length(rows, pool_size[0], 'valid',
+                                        pool_size[0])
+        cols = utils.conv_output_length(cols, pool_size[1], 'valid',
+                                        pool_size[1])
+        return [input_shape[0], rows, cols, input_shape[3]]