From 7074d6f77a43ba96d43a3adcd9397cf569bbe08c Mon Sep 17 00:00:00 2001
From: Luc Giffon <luc.giffon@lis-lab.fr>
Date: Thu, 22 Nov 2018 15:58:39 +0100
Subject: [PATCH] WIP fonctions noyau et deepstrom layer en keras
---
skluc/main/keras_/__init__.py | 0
skluc/main/keras_/kernel.py | 31 ++++
.../keras_/kernel_approximation/__init__.py | 2 +
.../kernel_approximation/nystrom_layer.py | 137 ++++++++++++++++++
4 files changed, 170 insertions(+)
create mode 100644 skluc/main/keras_/__init__.py
create mode 100644 skluc/main/keras_/kernel.py
create mode 100644 skluc/main/keras_/kernel_approximation/__init__.py
create mode 100644 skluc/main/keras_/kernel_approximation/nystrom_layer.py
diff --git a/skluc/main/keras_/__init__.py b/skluc/main/keras_/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/skluc/main/keras_/kernel.py b/skluc/main/keras_/kernel.py
new file mode 100644
index 0000000..653be6c
--- /dev/null
+++ b/skluc/main/keras_/kernel.py
@@ -0,0 +1,31 @@
+import tensorflow as tf
+
+from skluc.main.tensorflow_.utils import replace_nan
+
+
+def keras_linear_kernel(args):
+ X = args[0]
+ Y = args[1]
+ X = tf.nn.l2_normalize(X, axis=-1)
+ Y = tf.nn.l2_normalize(Y, axis=-1)
+ return tf.matmul(X, tf.transpose(Y))
+
+def keras_chi_square_CPD(args):
+ X = args[0]
+ Y = args[1]
+ # X = tf.nn.l2_normalize(X, axis=-1)
+ # Y = tf.nn.l2_normalize(Y, axis=-1)
+ # the drawing of the matrix X expanded looks like a wall
+ wall = tf.expand_dims(X, axis=1)
+ # the drawing of the matrix Y expanded looks like a floor
+ floor = tf.expand_dims(Y, axis=0)
+ numerator = tf.square((wall - floor))
+ denominator = wall + floor
+ quotient = numerator / denominator
+ quotient_without_nan = replace_nan(quotient)
+ K = - tf.reduce_sum(quotient_without_nan, axis=2)
+ return K
+
+
+if __name__ == '__main__':
+ a = tf.Constant(value=0)
\ No newline at end of file
diff --git a/skluc/main/keras_/kernel_approximation/__init__.py b/skluc/main/keras_/kernel_approximation/__init__.py
new file mode 100644
index 0000000..3a65679
--- /dev/null
+++ b/skluc/main/keras_/kernel_approximation/__init__.py
@@ -0,0 +1,2 @@
+from skluc.main.tensorflow_.kernel_approximation import fastfood_layer as fastfood_layer
+from skluc.main.tensorflow_.kernel_approximation import nystrom_layer as nystrom_layer
diff --git a/skluc/main/keras_/kernel_approximation/nystrom_layer.py b/skluc/main/keras_/kernel_approximation/nystrom_layer.py
new file mode 100644
index 0000000..1c2458c
--- /dev/null
+++ b/skluc/main/keras_/kernel_approximation/nystrom_layer.py
@@ -0,0 +1,137 @@
+"""
+Convnet with nystrom approximation of the feature map.
+
+"""
+import time as t
+
+import numpy as np
+import keras
+from sklearn.metrics.pairwise import rbf_kernel, linear_kernel, additive_chi2_kernel, chi2_kernel, laplacian_kernel
+
+import skluc.main.data.mldatasets as dataset
+from skluc.main.tensorflow_.kernel import tf_rbf_kernel, tf_chi_square_CPD, tf_chi_square_CPD_exp, tf_laplacian_kernel, tf_linear_kernel
+from skluc.main.utils import logger
+import tensorflow as tf
+
+class DeepstromLayerEndToEnd(keras.layers.Layer):
+ def __init__(self,
+ subsample_size,
+ kernel_name,
+ out_dim=None,
+ activation=None,
+ sum_of_kernels=False,
+ stack_of_kernels=False,
+ kernel_dict={}
+ ):
+
+ def init_kernel():
+ if kernel_name == "rbf":
+ kernel_fct = rbf_kernel
+ tf_kernel_fct = tf_rbf_kernel
+ elif kernel_name == "linear":
+ kernel_fct = linear_kernel
+ tf_kernel_fct = tf_linear_kernel
+ elif kernel_name == "chi2_cpd":
+ kernel_fct = additive_chi2_kernel
+ tf_kernel_fct = tf_chi_square_CPD
+ elif kernel_name == "chi2_exp_cpd":
+ kernel_fct = chi2_kernel
+ tf_kernel_fct = tf_chi_square_CPD_exp
+ elif kernel_name == "chi2_pd":
+ raise NotImplementedError("Bien verifier que ce code ne fait pas bordel")
+ elif kernel_name == "laplacian":
+ tf_kernel_fct = tf_laplacian_kernel
+ kernel_fct = laplacian_kernel
+ else:
+ raise ValueError("Unknown kernel name: {}".format(kernel_name))
+ return kernel_name, kernel_fct, tf_kernel_fct, kernel_dict
+
+ def init_output_dim(subsample_size):
+ if out_dim is not None and out_dim > subsample_size:
+ logger.debug("Output dim is greater than deepstrom subsample size. Aborting.")
+ exit()
+ elif out_dim is None:
+ return subsample_size
+ else:
+ return out_dim
+
+ def init_activation():
+ if activation == "tan":
+ activation_fct = tf.nn.tanh
+ elif activation == "relu":
+ activation_fct = tf.nn.relu
+ else:
+ activation_fct = activation
+
+ return activation_fct
+
+ super().__init__()
+
+ self.__subsample_size = subsample_size
+
+ self.__sum_of_kernels = sum_of_kernels
+ self.__stack_of_kernels = stack_of_kernels
+
+ self.__kernel_name, self.__kernel_fct, self.__tf_kernel_fct, self.__kernel_dict = init_kernel()
+ self.__output_dim = init_output_dim(self.__subsample_size)
+ self.__activation = init_activation()
+ self.__W_matrix = None
+
+ logger.info("Selecting deepstrom layer function with "
+ "subsample size = {}, "
+ "output_dim = {}, "
+ "{} activation function "
+ "and kernel = {}"
+ .format(self.__subsample_size,
+ self.__output_dim,
+ "with" if self.__activation else "without",
+ self.__kernel_name))
+
+ def build(self, input_shape):
+ if self.__output_dim != 0:
+ # outputdim == 0 means there is no W matrix and the kernel vector is directly added as input to
+ # the next layer
+ self.__W_matrix = self.add_weight(
+ name="W_nystrom",
+ shape=[self.__subsample_size, self.__output_dim],
+ initializer='glorot_uniform',
+ trainable=True
+ )
+ super(DeepstromLayerEndToEnd, self).build(input_shape)
+
+ @property
+ def output_dim(self):
+ return self.__output_dim
+
+ def call(self, inputs, **kwargs):
+ if type(inputs) is not list:
+ raise ValueError("Inputs of layer deepstrom should be a list")
+ if len(inputs[0].shape) != 2:
+ raise ValueError(f"Input x should be 2D but it is {len(inputs[0].shape)}D")
+ if len(inputs[1].shape) != 2:
+ raise ValueError(f"Input subsample should be 2D but it is {len(inputs[1].shape)}D")
+ if inputs[1].shape[0] != self.__subsample_size:
+ raise ValueError(f"Subsample should be of size {self.__subsample_size}")
+ if inputs[0][0].shape[0] != inputs[1][0].shape[0]:
+ raise ValueError(f"Input and subsample should have the same dimension")
+
+ input_x = inputs[0]
+ input_sub = inputs[1]
+ input_x = tf.nn.l2_normalize(input_x, axis=-1)
+ input_sub = tf.nn.l2_normalize(input_sub, axis=-1)
+ with tf.name_scope("NystromLayer"):
+ with tf.name_scope("kernel_vec"):
+ kernel_vector = self.__tf_kernel_fct(input_x, input_sub, **self.__kernel_dict)
+ tf.summary.histogram("kernel_vector", kernel_vector)
+
+ if self.__output_dim != 0:
+ out = tf.matmul(kernel_vector, self.__W_matrix)
+ tf.summary.histogram("W_matrix", self.__W_matrix)
+ else:
+ out = kernel_vector
+ if self.__activation is not None:
+ out = self.__activation(out)
+ return out
+
+ def compute_output_shape(self, input_shape):
+ return (input_shape[0], self.output_dim)
\ No newline at end of file
--
GitLab