Adapt tensorflow functions/classes to keras

Create FastfoodLayer class (WIP fast food layer function)
Update NystromLayer class, using keras functions instead of tensorflow ones
Improve chi2 and linear kernel functions + create exp chi2 cpd
Update model definition functions, using keras functions instead of tensorflow ones

skluc/main/keras_/kernel.py

-import tensorflow as tf
+# import tensorflow as tftf
+import keras.backend as K
+from keras.activations import tanh
 
 from skluc.main.tensorflow_.utils import replace_nan
 
 
-def keras_linear_kernel(args):
+def keras_linear_kernel(args, normalize=True):
     X = args[0]
     Y = args[1]
-    X = tf.nn.l2_normalize(X, axis=-1)
-    Y = tf.nn.l2_normalize(Y, axis=-1)
-    return tf.matmul(X, tf.transpose(Y))
+    if normalize:
+        X = K.l2_normalize(X, axis=-1)
+        Y = K.l2_normalize(Y, axis=-1)
+    return K.dot(X, K.transpose(Y))
 
-def keras_chi_square_CPD(args):
+
+def keras_chi_square_CPD(args, epsilon=None, tanh_activation=True, normalize=True):
     X = args[0]
     Y = args[1]
-    # X = tf.nn.l2_normalize(X, axis=-1)
-    # Y = tf.nn.l2_normalize(Y, axis=-1)
+    if normalize:
+        X = K.l2_normalize(X, axis=-1)
+        Y = K.l2_normalize(Y, axis=-1)
     # the drawing of the matrix X expanded looks like a wall
-    wall = tf.expand_dims(X, axis=1)
+    wall = K.expand_dims(X, axis=1)
     # the drawing of the matrix Y expanded looks like a floor
-    floor = tf.expand_dims(Y, axis=0)
-    numerator = tf.square((wall - floor))
+    floor = K.expand_dims(Y, axis=0)
+    numerator = K.square((wall - floor))
     denominator = wall + floor
-    quotient = numerator / denominator
+    if epsilon is not None:
+        quotient = numerator / (denominator + epsilon)
+    else:
+        quotient = numerator / denominator
     quotient_without_nan = replace_nan(quotient)
-    K = - tf.reduce_sum(quotient_without_nan, axis=2)
-    return K
-
-def chi2_kernel(args):
-    x = args[0]
-    y = tf.concat(args[1:], 0)
-
-    x = tf.nn.l2_normalize(x, axis=-1)
-    y = tf.nn.l2_normalize(y, axis=-1)
-    # the drawing of the matrix X expanded looks like a wall
-    wall = tf.expand_dims(x, axis=1)
-    # the drawing of the matrix Y expanded looks like a floor
-    floor = tf.expand_dims(y, axis=0)
-    numerator = tf.square(tf.subtract(wall, floor))
-    denominator = tf.add(wall, floor) + 0.001
-    quotient = numerator / denominator
-    quotient_without_nan = quotient #replace_nan(quotient)
-
-    K = - tf.reduce_sum(quotient_without_nan, axis=2)
-    return tf.tanh(K)
-
-if __name__ == '__main__':
-    a = tf.Constant(value=0)
\ No newline at end of file
+    result = - K.sum(quotient_without_nan, axis=2)
+    if tanh_activation:
+        return tanh(result)
+    else:
+        return result
+
+
+def keras_chi_square_CPD_exp(X, Y, gamma):
+    K = keras_chi_square_CPD(X, Y)
+    K *= gamma
+    return K.exp(K)
+
+# def chi2_kernel(args):
+#     x = args[0]
+#     y = tf.concat(args[1:], 0)
+#
+#     x = tf.nn.l2_normalize(x, axis=-1)
+#     y = tf.nn.l2_normalize(y, axis=-1)
+#     # the drawing of the matrix X expanded looks like a wall
+#     wall = tf.expand_dims(x, axis=1)
+#     # the drawing of the matrix Y expanded looks like a floor
+#     floor = tf.expand_dims(y, axis=0)
+#     numerator = tf.square(tf.subtract(wall, floor))
+#     denominator = tf.add(wall, floor) + 0.001
+#     quotient = numerator / denominator
+#     quotient_without_nan = quotient #replace_nan(quotient)
+#
+#     K = - tf.reduce_sum(quotient_without_nan, axis=2)
+#     return tf.tanh(K)

skluc/main/keras_/kernel_approximation/fastfood_layer.py

+import keras
+import keras.backend as K
+import numpy as np
+import scipy.linalg
+import scipy.stats
+import tensorflow as tf
+import tensorflow.summary as tfsum
+
+
+# --- Fast Food Naive --- #
+
+
+def G_variable(shape, trainable=False):
+    """
+    Return a Gaussian Random matrix converted into Tensorflow Variable.
+
+    :param shape: The shape of the matrix (number of fastfood stacks (v), dimension of the input space (d))
+    :type shape: int or tuple of int (tuple size = 2)
+    :return: K.variable object containing the matrix, The norm2 of each line (np.array of float)
+    """
+    assert type(shape) == int or (type(shape) == tuple and len(shape) == 2)
+    G = np.random.normal(size=shape).astype(np.float32)
+    G_norms = np.linalg.norm(G, ord=2, axis=1)
+    if trainable:
+        return K.variable(G, name="G"), G_norms
+    else:
+        return K.constant(G, name="G"), G_norms
+
+
+def B_variable(shape, trainable=False):
+    """
+    Return a random matrix of -1 and 1 picked uniformly and converted into Tensorflow Variable.
+
+    :param shape: The shape of the matrix (number of fastfood stacks (v), dimension of the input space (d))
+    :type shape: int or tuple of int (tuple size = 2)
+    :return: K.variable object containing the matrix
+    """
+    assert type(shape) == int or (type(shape) == tuple and len(shape) == 2)
+    B = np.random.choice([-1, 1], size=shape, replace=True).astype(np.float32)
+    if trainable:
+        return K.variable(B, name="B")
+    else:
+        return K.constant(B, name="B")
+
+
+def P_variable(d, nbr_stack):
+    """
+    Return a permutation matrix converted into Tensorflow Variable.
+
+    :param d: The width of the matrix (dimension of the input space)
+    :type d: int
+    :param nbr_stack: The height of the matrix (nbr_stack x d is the dimension of the output space)
+    :type nbr_stack: int
+    :return: K.variable object containing the matrix
+    """
+    idx = np.hstack([(i * d) + np.random.permutation(d) for i in range(nbr_stack)])
+    P = np.random.permutation(np.eye(N=nbr_stack * d))[idx].astype(np.float32)
+    return K.constant(P, name="P")
+
+
+def H_variable(d):
+    """
+    Return an Hadamard matrix converted into Tensorflow Variable.
+
+    d must be a power of two.
+
+    :param d: The size of the Hadamard matrix (dimension of the input space).
+    :type d: int
+    :return: K.variable object containing the diagonal and not trainable
+    """
+    H = build_hadamard(d).astype(np.float32)
+    return K.constant(H, name="H")
+
+
+def S_variable(shape, G_norms, trainable=False):
+    """
+    Return a scaling matrix of random values picked from a chi distribution.
+
+    The values are re-scaled using the norm of the associated Gaussian random matrix G. The associated Gaussian
+    vectors are the ones generated by the `G_variable` function.
+
+    :param shape: The shape of the matrix (number of fastfood stacks (v), dimension of the input space (d))
+    :type shape: int or tuple of int (tuple size = 2)
+    :param G_norms: The norms of the associated Gaussian random matrices G.
+    :type G_norms: np.array of floats
+    :return: K.variable object containing the matrix.
+    """
+    S = np.multiply((1 / G_norms.reshape((-1, 1))), scipy.stats.chi.rvs(shape[1], size=shape).astype(np.float32))
+    if trainable:
+        return K.variable(S, name="S")
+    else:
+        return K.constant(S, name="S")
+
+
+def fastfood_layer(conv_out, sigma, nbr_stack=1, trainable=False):
+    """
+    Return a fastfood transform op compatible with tensorflow graph.
+
+    Implementation largely inspired from https://gist.github.com/dougalsutherland/1a3c70e57dd1f64010ab .
+
+    See:
+    "Fastfood | Approximating Kernel Expansions in Loglinear Time" by
+    Quoc Le, Tamas Sarl and Alex Smola.
+
+    :param conv_out: the input of the op
+    :param sigma: bandwith of the gaussian distribution
+    :param nbr_stack: number of fast food stacks
+    :param trainable: the diagonal matrices are trainable or not
+    :return: the output of the fastfood transform
+    """
+    raise NotImplementedError("fastfood_layer function is not yet correctly implemented in keras")
+    with K.name_scope("fastfood" + "_sigma-" + str(sigma)):
+        init_dim = np.prod([s.value for s in conv_out.shape if s.value is not None])
+        final_dim = int(dimensionality_constraints(init_dim))
+        padding = final_dim - init_dim
+        conv_out2 = K.reshape(conv_out, [-1, init_dim])
+        paddings = K.constant([[0, 0], [0, padding]])
+        conv_out2 = K.pad(conv_out2, paddings, "CONSTANT")
+
+        G, G_norm = G_variable((nbr_stack, final_dim), trainable=trainable)
+        tfsum.histogram("weights_G", G)
+        B = B_variable((nbr_stack, final_dim), trainable=trainable)
+        tfsum.histogram("weights_B", B)
+        H = H_variable(final_dim)
+        tfsum.histogram("weights_H", H)
+        P = P_variable(final_dim, nbr_stack)
+        tfsum.histogram("weights_P", P)
+        S = S_variable((nbr_stack, final_dim), G_norm, trainable=trainable)
+        tfsum.histogram("weights_S", S)
+
+        conv_out2 = K.reshape(conv_out2, (1, -1, 1, final_dim))
+        h_ff1 = conv_out2 * B
+        h_ff1 = K.reshape(h_ff1, (-1, final_dim))
+        h_ff2 = K.dot(h_ff1, H)
+        h_ff2 = K.reshape(h_ff2, (-1, final_dim * nbr_stack))
+        h_ff3 = K.dot(h_ff2, P)
+        h_ff4 = K.reshape(h_ff3, (-1, final_dim * nbr_stack)) * K.reshape(G, (-1, final_dim * nbr_stack))
+        h_ff4 = K.reshape(h_ff4, (-1, final_dim))
+        h_ff5 = K.dot(h_ff4, H)
+
+        h_ff6 = (1. / (sigma * np.sqrt(final_dim))) * K.reshape(h_ff5, (-1, final_dim * nbr_stack)) * K.reshape(S, (-1, final_dim * nbr_stack))
+        h_ff7_1 = K.cos(h_ff6)
+        h_ff7_2 = K.sin(h_ff6)
+        h_ff7 = K.sqrt(float(1 / final_dim)) * K.concatenate([h_ff7_1, h_ff7_2], axis=1)
+    return h_ff7
+
+
+# --- Hadamard utils --- #
+
+
+def dimensionality_constraints(d):
+    """
+    Enforce d to be a power of 2
+
+    :param d: the original dimension
+    :return: the final dimension
+    """
+    if not is_power_of_two(d):
+        # find d that fulfills 2^l
+        d = np.power(2, np.floor(np.log2(d)) + 1)
+    return d
+
+
+def is_power_of_two(input_integer):
+    """ Test if an integer is a power of two. """
+    if input_integer == 1:
+        return False
+    return input_integer != 0 and ((input_integer & (input_integer - 1)) == 0)
+
+
+def build_hadamard(n_neurons):
+    return scipy.linalg.hadamard(n_neurons)
+
+
+class FastFoodLayer(keras.layers.Layer):
+    def __init__(self, sigma, nbr_stack, trainable=True, **kwargs):
+        super().__init__(**kwargs)
+        self.__sigma = sigma
+        self.__nbr_stack = nbr_stack
+        self.__trainable = trainable
+
+        self.__init_dim = None
+        self.__final_dim = None
+
+    def build(self, input_shape):
+        with K.name_scope("fastfood" + "_sigma-" + str(self.__sigma)):
+            self.__init_dim = np.prod([s for s in input_shape if s is not None])
+            self.__final_dim = int(dimensionality_constraints(self.__init_dim))
+            self.num_outputs = None
+
+            G, G_norm = G_variable((self.__nbr_stack, self.__final_dim))
+            self.__G = self.add_weight(
+                name="G",
+                shape=(self.__nbr_stack, self.__final_dim),
+                initializer=lambda *args, **kwargs: G,
+                trainable=self.__trainable
+            )
+
+            B = B_variable((self.__nbr_stack, self.__final_dim))
+            self.__B = self.add_weight(
+                name="B",
+                shape=(self.__nbr_stack, self.__final_dim),
+                initializer=lambda *args, **kwargs: B,
+                trainable=self.__trainable
+            )
+
+            H = H_variable(self.__final_dim)
+            self.__H = self.add_weight(
+                name="H",
+                shape=(self.__final_dim, self.__final_dim),
+                initializer=lambda *args, **kwargs: H,
+                trainable=False
+            )
+
+            P = P_variable(self.__final_dim, self.__nbr_stack)
+            self.__P = self.add_weight(
+                name="P",
+                shape=(self.__final_dim * self.__nbr_stack, self.__final_dim * self.__nbr_stack),
+                initializer=lambda *args, **kwargs: P,
+                trainable=False
+            )
+
+            S = S_variable((self.__nbr_stack, self.__final_dim), G_norm)
+            self.__S = self.add_weight(
+                name="S",
+                shape=(self.__final_dim * self.__nbr_stack, self.__final_dim * self.__nbr_stack),
+                initializer=lambda *args, **kwargs: S,
+                trainable=self.__trainable
+            )
+
+        self.num_outputs = self.__final_dim * self.__nbr_stack
+
+    def call(self, input, **kwargs):
+        padding = self.__final_dim - self.__init_dim
+        conv_out2 = K.reshape(input, [-1, self.__init_dim])
+        paddings = K.constant([[0, 0], [0, padding]], dtype=np.int32)
+        conv_out2 = tf.pad(conv_out2, paddings, "CONSTANT")
+        conv_out2 = K.reshape(conv_out2, (1, -1, 1, self.__final_dim))
+        h_ff1 = conv_out2 * self.__B
+        h_ff1 = K.reshape(h_ff1, (-1, self.__final_dim))
+        h_ff2 = K.dot(h_ff1, self.__H)
+        h_ff2 = K.reshape(h_ff2, (-1, self.__final_dim * self.__nbr_stack))
+        h_ff3 = K.dot(h_ff2, self.__P)
+        h_ff4 = K.reshape(h_ff3, (-1, self.__final_dim * self.__nbr_stack)) * K.reshape(self.__G, (-1, self.__final_dim * self.__nbr_stack))
+        h_ff4 = K.reshape(h_ff4, (-1, self.__final_dim))
+        h_ff5 = K.dot(h_ff4, self.__H)
+
+        h_ff6 = (1 / (self.__sigma * np.sqrt(self.__final_dim))) * K.reshape(h_ff5, (-1, self.__final_dim * self.__nbr_stack)) * K.reshape(self.__S, (-1, self.__final_dim * self.__nbr_stack))
+        h_ff7_1 = K.cos(h_ff6)
+        h_ff7_2 = K.sin(h_ff6)
+        h_ff7 = np.sqrt(float(1 / self.__final_dim)) * K.concatenate([h_ff7_1, h_ff7_2], axis=1)
+        return h_ff7
+
+    def compute_output_shape(self, input_shape):
+        return (input_shape[0], 2 * input_shape[1] * self.__nbr_stack)

skluc/main/keras_/kernel_approximation/nystrom_layer.py

@@ -2,16 +2,16 @@
 Convnet with nystrom approximation of the feature map.
 
 """
-import time as t
 
-import numpy as np
 import keras
+import keras.backend as K
+import tensorflow.summary as tfsum
 from sklearn.metrics.pairwise import rbf_kernel, linear_kernel, additive_chi2_kernel, chi2_kernel, laplacian_kernel
 
-import skluc.main.data.mldatasets as dataset
-from skluc.main.tensorflow_.kernel import tf_rbf_kernel, tf_chi_square_CPD, tf_chi_square_CPD_exp, tf_laplacian_kernel, tf_linear_kernel
+from skluc.main.keras_.kernel import keras_chi_square_CPD, keras_linear_kernel
+from skluc.main.tensorflow_.kernel import tf_rbf_kernel, tf_chi_square_CPD_exp, tf_laplacian_kernel
 from skluc.main.utils import logger
-import tensorflow as tf
+
 
 class DeepstromLayerEndToEnd(keras.layers.Layer):
     def __init__(self,
@@ -25,25 +25,30 @@ class DeepstromLayerEndToEnd(keras.layers.Layer):
                  ):
 
         def init_kernel():
-            if kernel_name == "rbf":
-                kernel_fct = rbf_kernel
-                tf_kernel_fct = tf_rbf_kernel
-            elif kernel_name == "linear":
-                kernel_fct = linear_kernel
-                tf_kernel_fct = tf_linear_kernel
-            elif kernel_name == "chi2_cpd":
-                kernel_fct = additive_chi2_kernel
-                tf_kernel_fct = tf_chi_square_CPD
-            elif kernel_name == "chi2_exp_cpd":
-                kernel_fct = chi2_kernel
-                tf_kernel_fct = tf_chi_square_CPD_exp
-            elif kernel_name == "chi2_pd":
-                raise NotImplementedError("Bien verifier que ce code ne fait pas bordel")
-            elif kernel_name == "laplacian":
-                tf_kernel_fct = tf_laplacian_kernel
-                kernel_fct = laplacian_kernel
-            else:
-                raise ValueError("Unknown kernel name: {}".format(kernel_name))
+            try:
+                if kernel_name == "rbf":
+                    raise NotImplementedError
+                    kernel_fct = rbf_kernel
+                    tf_kernel_fct = tf_rbf_kernel
+                elif kernel_name == "linear":
+                    kernel_fct = linear_kernel
+                    tf_kernel_fct = keras_linear_kernel
+                elif kernel_name == "chi2_cpd":
+                    kernel_fct = additive_chi2_kernel
+                    tf_kernel_fct = keras_chi_square_CPD
+                elif kernel_name == "chi2_exp_cpd":
+                    raise NotImplementedError
+                    kernel_fct = chi2_kernel
+                    tf_kernel_fct = tf_chi_square_CPD_exp
+                elif kernel_name == "chi2_pd":
+                    raise NotImplementedError  # todo Bien verifier que ce code ne fait pas bordel
+                elif kernel_name == "laplacian":
+                    tf_kernel_fct = tf_laplacian_kernel
+                    kernel_fct = laplacian_kernel
+                else:
+                    raise ValueError("Unknown kernel name: {}".format(kernel_name))
+            except NotImplementedError:
+                raise NotImplementedError(f"kernel {kernel_name} has not been implemented with keras yet")
             return kernel_name, kernel_fct, tf_kernel_fct, kernel_dict
 
         def init_output_dim(subsample_size):
@@ -57,9 +62,9 @@ class DeepstromLayerEndToEnd(keras.layers.Layer):
 
         def init_activation():
             if activation == "tan":
-                activation_fct = tf.nn.tanh
+                activation_fct = keras.activations.tanh
             elif activation == "relu":
-                activation_fct = tf.nn.relu
+                activation_fct = keras.activations.relu
             else:
                 activation_fct = activation
 
@@ -117,16 +122,16 @@ class DeepstromLayerEndToEnd(keras.layers.Layer):
 
         input_x = inputs[0]
         input_sub = inputs[1]
-        input_x = tf.nn.l2_normalize(input_x, axis=-1)
-        input_sub = tf.nn.l2_normalize(input_sub, axis=-1)
-        with tf.name_scope("NystromLayer"):
-            with tf.name_scope("kernel_vec"):
+        # input_x = K.l2_normalize(input_x, axis=-1)
+        # input_sub = K.l2_normalize(input_sub, axis=-1)
+        with K.name_scope("NystromLayer"):
+            with K.name_scope("kernel_vec"):
                 kernel_vector = self.__tf_kernel_fct(input_x, input_sub, **self.__kernel_dict)
-                tf.summary.histogram("kernel_vector", kernel_vector)
+                tfsum.histogram("kernel_vector", kernel_vector)
 
             if self.__output_dim != 0:
-                out = tf.matmul(kernel_vector, self.__W_matrix)
-                tf.summary.histogram("W_matrix", self.__W_matrix)
+                out = K.dot(kernel_vector, self.__W_matrix)
+                tfsum.histogram("W_matrix", self.__W_matrix)
             else:
                 out = kernel_vector
         if self.__activation is not None:

skluc/main/keras_/models.py

+from keras.initializers import he_normal
+from keras.layers import Conv2D, MaxPooling2D, Flatten, BatchNormalization, Activation
+from keras.models import Sequential
+from keras.regularizers import l2
+
+
+def build_lenet_model(input_shape):
+    model = Sequential()
+    model.add(
+        Conv2D(6, (5, 5), padding='valid', activation='relu', kernel_initializer=he_normal(), input_shape=input_shape))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2)))
+    model.add(Conv2D(16, (5, 5), padding='valid', activation='relu', kernel_initializer=he_normal()))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2)))
+    model.add(Flatten())
+    return model
+
+
+def build_steph_model(input_shape):
+    model = Sequential()
+    model.add(Conv2D(64, kernel_size=(4, 4), strides=(2, 2), padding='same', activation=None, input_shape=input_shape))
+    model.add(Activation('relu'))
+    model.add(Conv2D(128, kernel_size=(4, 4), strides=(2, 2), padding='same', activation=None))
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, kernel_size=(4, 4), strides=(2, 2), padding='same', activation=None))
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, kernel_size=(3, 3), strides=(1, 1), padding='same', activation=None))
+    model.add(BatchNormalization())
+    model.add(MaxPooling2D((2, 2)))
+    model.add(Activation('relu'))
+    model.add(Flatten())
+    return model
+
+
+def build_vgg19_model(input_shape, weight_decay=0.0001):
+    model = Sequential()
+
+    # Block 1
+    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block1_conv1', input_shape=input_shape))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block1_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))
+
+    # Block 2
+    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block2_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block2_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))
+
+    # Block 3
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block3_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block3_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block3_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block3_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))
+
+    # Block 4
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block4_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block4_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block4_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block4_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))
+
+    # Block 5
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block5_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block5_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block5_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     kernel_initializer=he_normal(), name='block5_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool'))
+
+    model.add(Flatten(name='flatten'))
+
+    return model
+
+
+def build_vgg19_model_glorot(input_shape, weight_decay=0.0001):
+    model = Sequential()
+
+    # Block 1
+    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block1_conv1', input_shape=input_shape))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(64, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block1_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool'))
+
+    # Block 2
+    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block2_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(128, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block2_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool'))
+
+    # Block 3
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block3_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block3_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block3_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(256, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block3_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool'))
+
+    # Block 4
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block4_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block4_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block4_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block4_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool'))
+
+    # Block 5
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block5_conv1'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block5_conv2'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block5_conv3'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(Conv2D(512, (3, 3), padding='same', kernel_regularizer=l2(weight_decay),
+                     name='block5_conv4'))
+    model.add(BatchNormalization())
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool'))
+
+    model.add(Flatten(name='flatten'))
+
+    return model