v0.2.0

Support different initializers for different embedding weights and loading pretrained embeddings

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,9 +18,9 @@ Steps to reproduce the behavior:
 4. See error
 
 **Operating environment(运行环境):**
- - python version [e.g. 3.4, 3.6]
- - tensorflow version [e.g. 1.4.0, 1.12.0]
- - deepmatch version [e.g. 0.1.1,]
+ - python version [e.g. 3.6, 3.7]
+ - tensorflow version [e.g. 1.4.0, 1.14.0, 2.3.0]
+ - deepmatch version [e.g. 0.2.0,]
 
 **Additional context**
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/question.md

@@ -17,4 +17,4 @@ Add any other context about the problem here.
 **Operating environment(运行环境):**
  - python version [e.g. 3.6]
  - tensorflow version [e.g. 1.4.0,]
- - deepmatch version [e.g. 0.1.1,]
+ - deepmatch version [e.g. 0.2.0,]

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.5,3.6,3.7]
-        tf-version: [1.4.0,1.14.0,2.1.0,2.2.0]
+        tf-version: [1.4.0,1.14.0,2.1.0,2.2.0,2.3.0]
 
         exclude:
           - python-version: 3.7

README.md

@@ -67,4 +67,4 @@ Please follow our wechat to join group:
 - 公众号：**浅梦的学习笔记**  
 - wechat ID: **deepctrbot**
 
-  ![wechat](./docs/pics/weichennote.png)
+  ![wechat](./docs/pics/code.png)

deepmatch/__init__.py

@@ -1,4 +1,4 @@
 from .utils import check_version
 
-__version__ = '0.1.3'
+__version__ = '0.2.0'
 check_version(__version__)

deepmatch/inputs.py

@@ -1,14 +1,17 @@
 from itertools import chain
-from deepctr.inputs import SparseFeat,VarLenSparseFeat,create_embedding_matrix,embedding_lookup,get_dense_input,varlen_embedding_lookup,get_varlen_pooling_list,mergeDict
 
-def input_from_feature_columns(features, feature_columns, l2_reg, init_std, seed, prefix='', seq_mask_zero=True,
-                               support_dense=True, support_group=False,embedding_matrix_dict=None):
+from deepctr.feature_column import SparseFeat, VarLenSparseFeat, create_embedding_matrix, embedding_lookup, \
+    get_dense_input, varlen_embedding_lookup, get_varlen_pooling_list, mergeDict
+
+
+def input_from_feature_columns(features, feature_columns, l2_reg, seed, prefix='', seq_mask_zero=True,
+                               support_dense=True, support_group=False, embedding_matrix_dict=None):
     sparse_feature_columns = list(
         filter(lambda x: isinstance(x, SparseFeat), feature_columns)) if feature_columns else []
     varlen_sparse_feature_columns = list(
         filter(lambda x: isinstance(x, VarLenSparseFeat), feature_columns)) if feature_columns else []
-    if embedding_matrix_dict is  None:
-        embedding_matrix_dict = create_embedding_matrix(feature_columns, l2_reg, init_std, seed, prefix=prefix,
+    if embedding_matrix_dict is None:
+        embedding_matrix_dict = create_embedding_matrix(feature_columns, l2_reg, seed, prefix=prefix,
                                                         seq_mask_zero=seq_mask_zero)
 
     group_sparse_embedding_dict = embedding_lookup(embedding_matrix_dict, features, sparse_feature_columns)
@@ -22,4 +25,4 @@ def input_from_feature_columns(features, feature_columns, l2_reg, init_std, seed
     group_embedding_dict = mergeDict(group_sparse_embedding_dict, group_varlen_sparse_embedding_dict)
     if not support_group:
         group_embedding_dict = list(chain.from_iterable(group_embedding_dict.values()))
-    return group_embedding_dict, dense_value_list
+    return group_embedding_dict, dense_value_list

deepmatch/layers/__init__.py

@@ -1,28 +1,29 @@
 from deepctr.layers import custom_objects
 from deepctr.layers.utils import reduce_sum
 
-from .core import PoolingLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayer,EmbeddingIndex
-from ..utils import sampledsoftmaxloss
-from .interaction import DotAttention, ConcatAttention, SoftmaxWeightedSum, AttentionSequencePoolingLayer, SelfAttention,\
+from .core import PoolingLayer, Similarity, LabelAwareAttention, CapsuleLayer, SampledSoftmaxLayer, EmbeddingIndex
+from .interaction import DotAttention, ConcatAttention, SoftmaxWeightedSum, AttentionSequencePoolingLayer, \
+    SelfAttention, \
     SelfMultiHeadAttention, UserAttention
 from .sequence import DynamicMultiRNN
+from ..utils import sampledsoftmaxloss
 
 _custom_objects = {'PoolingLayer': PoolingLayer,
                    'Similarity': Similarity,
                    'LabelAwareAttention': LabelAwareAttention,
                    'CapsuleLayer': CapsuleLayer,
-                   'reduce_sum':reduce_sum,
-                   'SampledSoftmaxLayer':SampledSoftmaxLayer,
-                   'sampledsoftmaxloss':sampledsoftmaxloss,
-                   'EmbeddingIndex':EmbeddingIndex,
-                   'DotAttention':DotAttention,
-                   'ConcatAttention':ConcatAttention,
-                   'SoftmaxWeightedSum':SoftmaxWeightedSum,
-                   'AttentionSequencePoolingLayer':AttentionSequencePoolingLayer,
-                   'SelfAttention':SelfAttention,
-                   'SelfMultiHeadAttention':SelfMultiHeadAttention,
-                   'UserAttention':UserAttention,
-                    'DynamicMultiRNN':DynamicMultiRNN
+                   'reduce_sum': reduce_sum,
+                   'SampledSoftmaxLayer': SampledSoftmaxLayer,
+                   'sampledsoftmaxloss': sampledsoftmaxloss,
+                   'EmbeddingIndex': EmbeddingIndex,
+                   'DotAttention': DotAttention,
+                   'ConcatAttention': ConcatAttention,
+                   'SoftmaxWeightedSum': SoftmaxWeightedSum,
+                   'AttentionSequencePoolingLayer': AttentionSequencePoolingLayer,
+                   'SelfAttention': SelfAttention,
+                   'SelfMultiHeadAttention': SelfMultiHeadAttention,
+                   'UserAttention': UserAttention,
+                   'DynamicMultiRNN': DynamicMultiRNN
                    }
 
 custom_objects = dict(custom_objects, **_custom_objects)

deepmatch/layers/core.py

@@ -1,7 +1,9 @@
 import tensorflow as tf
+from deepctr.layers.activation import activation_layer
 from deepctr.layers.utils import reduce_max, reduce_mean, reduce_sum, concat_func, div, softmax
-from tensorflow.python.keras.initializers import RandomNormal, Zeros
+from tensorflow.python.keras.initializers import RandomNormal, Zeros, glorot_normal
 from tensorflow.python.keras.layers import Layer
+from tensorflow.python.keras.regularizers import l2
 
 
 class PoolingLayer(Layer):
@@ -216,21 +218,20 @@ def squash(inputs):
     return vec_squashed
 
 
-
-
 class EmbeddingIndex(Layer):
 
-    def __init__(self, index,**kwargs):
-        self.index =index
+    def __init__(self, index, **kwargs):
+        self.index = index
         super(EmbeddingIndex, self).__init__(**kwargs)
 
     def build(self, input_shape):
-
         super(EmbeddingIndex, self).build(
             input_shape)  # Be sure to call this somewhere!
+
     def call(self, x, **kwargs):
-       return tf.constant(self.index)
+        return tf.constant(self.index)
+
     def get_config(self, ):
         config = {'index': self.index, }
         base_config = super(EmbeddingIndex, self).get_config()
-        return dict(list(base_config.items()) + list(config.items()))
+        return dict(list(base_config.items()) + list(config.items()))

deepmatch/layers/sequence.py

@@ -24,27 +24,27 @@ def build(self, input_shape):
         if self.rnn_type == "LSTM":
             try:
                 single_cell = tf.nn.rnn_cell.BasicLSTMCell(self.num_units, forget_bias=self.forget_bias)
-            except:
+            except AttributeError:
                 single_cell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(self.num_units, forget_bias=self.forget_bias)
         elif self.rnn_type == "GRU":
             try:
                 single_cell = tf.nn.rnn_cell.GRUCell(self.num_units, forget_bias=self.forget_bias)
-            except:
+            except AttributeError:
                 single_cell = tf.compat.v1.nn.rnn_cell.GRUCell(self.num_units, forget_bias=self.forget_bias)
         else:
             raise ValueError("Unknown unit type %s!" % self.rnn_type)
         dropout = self.dropout if tf.keras.backend.learning_phase() == 1 else 0
         try:
             single_cell = tf.nn.rnn_cell.DropoutWrapper(cell=single_cell, input_keep_prob=(1.0 - dropout))
-        except:
+        except AttributeError:
             single_cell = tf.compat.v1.nn.rnn_cell.DropoutWrapper(cell=single_cell, input_keep_prob=(1.0 - dropout))
         cell_list = []
         for i in range(self.num_layers):
             residual = (i >= self.num_layers - self.num_residual_layers)
             if residual:
                 try:
                     single_cell_residual = tf.nn.rnn_cell.ResidualWrapper(single_cell)
-                except:
+                except AttributeError:
                     single_cell_residual = tf.compat.v1.nn.rnn_cell.ResidualWrapper(single_cell)
                 cell_list.append(single_cell_residual)
             else:
@@ -54,7 +54,7 @@ def build(self, input_shape):
         else:
             try:
                 self.final_cell = tf.nn.rnn_cell.MultiRNNCell(cell_list)
-            except:
+            except AttributeError:
                 self.final_cell = tf.compat.v1.nn.rnn_cell.MultiRNNCell(cell_list)
         super(DynamicMultiRNN, self).build(input_shape)
 
@@ -66,7 +66,7 @@ def call(self, input_list, mask=None, training=None):
                 rnn_output, hidden_state = tf.nn.dynamic_rnn(self.final_cell, inputs=rnn_input,
                                                              sequence_length=tf.squeeze(sequence_length),
                                                              dtype=tf.float32, scope=self.name)
-        except:
+        except AttributeError:
             with tf.name_scope("rnn"), tf.compat.v1.variable_scope("rnn", reuse=tf.compat.v1.AUTO_REUSE):
                 rnn_output, hidden_state = tf.compat.v1.nn.dynamic_rnn(self.final_cell, inputs=rnn_input,
                                                                        sequence_length=tf.squeeze(sequence_length),
@@ -86,6 +86,6 @@ def compute_output_shape(self, input_shape):
     def get_config(self, ):
         config = {'num_units': self.num_units, 'rnn_type': self.rnn_type, 'return_sequence': self.return_sequence,
                   'num_layers': self.num_layers,
-                  'num_residual_layers': self.num_residual_layers, 'dropout_rate': self.dropout}
+                  'num_residual_layers': self.num_residual_layers, 'dropout_rate': self.dropout, 'forget_bias':self.forget_bias}
         base_config = super(DynamicMultiRNN, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))

deepmatch/models/dssm.py

@@ -5,8 +5,8 @@
 Huang P S , He X , Gao J , et al. Learning deep structured semantic models for web search using clickthrough data[C]// Acm International Conference on Conference on Information & Knowledge Management. ACM, 2013.
 """
 
-from deepctr.inputs import build_input_features, combined_dnn_input, create_embedding_matrix
-from deepctr.layers.core import PredictionLayer, DNN
+from deepctr.feature_column import build_input_features, create_embedding_matrix
+from deepctr.layers import PredictionLayer, DNN, combined_dnn_input
 from tensorflow.python.keras.models import Model
 
 from ..inputs import input_from_feature_columns
@@ -16,7 +16,7 @@
 def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, 32),
          item_dnn_hidden_units=(64, 32),
          dnn_activation='tanh', dnn_use_bn=False,
-         l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, init_std=0.0001, seed=1024, metric='cos'):
+         l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, seed=1024, metric='cos'):
     """Instantiates the Deep Structured Semantic Model architecture.
 
     :param user_feature_columns: An iterable containing user's features used by  the model.
@@ -28,38 +28,37 @@ def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64,
     :param l2_reg_dnn: float. L2 regularizer strength applied to DNN
     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
     :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate.
-    :param init_std: float,to use as the initialize std of embedding vector
     :param seed: integer ,to use as random seed.
     :param metric: str, ``"cos"`` for  cosine  or  ``"ip"`` for inner product
     :return: A Keras model instance.
 
     """
 
     embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding,
-                                                    init_std, seed,
+                                                    seed=seed,
                                                     seq_mask_zero=True)
 
     user_features = build_input_features(user_feature_columns)
     user_inputs_list = list(user_features.values())
     user_sparse_embedding_list, user_dense_value_list = input_from_feature_columns(user_features,
                                                                                    user_feature_columns,
-                                                                                   l2_reg_embedding, init_std, seed,
+                                                                                   l2_reg_embedding, seed=seed,
                                                                                    embedding_matrix_dict=embedding_matrix_dict)
     user_dnn_input = combined_dnn_input(user_sparse_embedding_list, user_dense_value_list)
 
     item_features = build_input_features(item_feature_columns)
     item_inputs_list = list(item_features.values())
     item_sparse_embedding_list, item_dense_value_list = input_from_feature_columns(item_features,
                                                                                    item_feature_columns,
-                                                                                   l2_reg_embedding, init_std, seed,
+                                                                                   l2_reg_embedding, seed=seed,
                                                                                    embedding_matrix_dict=embedding_matrix_dict)
     item_dnn_input = combined_dnn_input(item_sparse_embedding_list, item_dense_value_list)
 
     user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
-                       dnn_use_bn, seed, )(user_dnn_input)
+                       dnn_use_bn, seed=seed)(user_dnn_input)
 
     item_dnn_out = DNN(item_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout,
-                       dnn_use_bn, seed)(item_dnn_input)
+                       dnn_use_bn, seed=seed)(item_dnn_input)
 
     score = Similarity(type=metric)([user_dnn_out, item_dnn_out])
 

deepmatch/models/fm.py

@@ -1,4 +1,4 @@
-from deepctr.inputs import build_input_features
+from deepctr.feature_column import build_input_features
 from deepctr.layers.core import PredictionLayer
 from deepctr.layers.utils import concat_func, reduce_sum
 from tensorflow.python.keras.layers import Lambda
@@ -8,36 +8,35 @@
 from ..layers.core import Similarity
 
 
-def FM(user_feature_columns, item_feature_columns, l2_reg_embedding=1e-6, init_std=0.0001, seed=1024, metric='cos'):
+def FM(user_feature_columns, item_feature_columns, l2_reg_embedding=1e-6, seed=1024, metric='cos'):
     """Instantiates the FM architecture.
 
     :param user_feature_columns: An iterable containing user's features used by  the model.
     :param item_feature_columns: An iterable containing item's features used by  the model.
     :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector
-    :param init_std: float,to use as the initialize std of embedding vector
     :param seed: integer ,to use as random seed.
     :param metric: str, ``"cos"`` for  cosine  or  ``"ip"`` for inner product
     :return: A Keras model instance.
 
     """
 
     embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding,
-                                                    init_std, seed,
+                                                    seed=seed,
                                                     seq_mask_zero=True)
 
     user_features = build_input_features(user_feature_columns)
     user_inputs_list = list(user_features.values())
     user_sparse_embedding_list, user_dense_value_list = input_from_feature_columns(user_features,
                                                                                    user_feature_columns,
-                                                                                   l2_reg_embedding, init_std, seed,
+                                                                                   l2_reg_embedding, seed=seed,
                                                                                    support_dense=False,
                                                                                    embedding_matrix_dict=embedding_matrix_dict)
 
     item_features = build_input_features(item_feature_columns)
     item_inputs_list = list(item_features.values())
     item_sparse_embedding_list, item_dense_value_list = input_from_feature_columns(item_features,
                                                                                    item_feature_columns,
-                                                                                   l2_reg_embedding, init_std, seed,
+                                                                                   l2_reg_embedding, seed=seed,
                                                                                    support_dense=False,
                                                                                    embedding_matrix_dict=embedding_matrix_dict)