From 88b6abebe15e91c56f04cc1a47db18216c63ee4e Mon Sep 17 00:00:00 2001
From: Luc Giffon <luc.giffon@lif.univ-mrs.fr>
Date: Mon, 19 Feb 2018 11:34:11 +0100
Subject: [PATCH] change in the dataset preprocessing + the nystrom layer has
now only one matrix W instead of D and V
---
main/nystrom/nystrom_approx.py | 94 ++++++++++------------------------
1 file changed, 28 insertions(+), 66 deletions(-)
diff --git a/main/nystrom/nystrom_approx.py b/main/nystrom/nystrom_approx.py
index 7013d5f..d0ebd6f 100644
--- a/main/nystrom/nystrom_approx.py
+++ b/main/nystrom/nystrom_approx.py
@@ -2,65 +2,27 @@
Convnet with nystrom approximation of the feature map.
"""
+import time as t
import tensorflow as tf
import numpy as np
import skluc.mldatasets as dataset
-from skluc.neural_networks import bias_variable, weight_variable, conv_relu_pool, get_next_batch
+from skluc.neural_networks import get_next_batch, classification_mnist, convolution_mnist
tf.logging.set_verbosity(tf.logging.ERROR)
-import time as t
-
-from sklearn.preprocessing import LabelBinarizer
-
-enc = LabelBinarizer()
-mnist = dataset.MnistDataset()
-mnist = mnist.load()
-X_train, Y_train = mnist["train"]
-X_train = np.array(X_train / 255)
-enc.fit(Y_train)
-Y_train = np.array(enc.transform(Y_train))
-X_test, Y_test = mnist["test"]
-X_test = np.array(X_test / 255)
-Y_test = np.array(enc.transform(Y_test))
-
-X_train = X_train.astype(np.float32)
-permut = np.random.permutation(X_train.shape[0])
val_size = 5000
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.to_one_hot()
+mnist.normalize()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
-X_val = X_train[permut[:val_size]]
-Y_val = Y_train[permut[:val_size]]
-X_train = X_train[permut[val_size:]]
-Y_train = Y_train[permut[val_size:]]
-X_test = X_test.astype(np.float32)
-Y_train = Y_train.astype(np.float32)
-Y_test = Y_test.astype(np.float32)
-
-NYSTROM_SAMPLE_SIZE = 500
-X_nystrom = X_train[np.random.permutation(NYSTROM_SAMPLE_SIZE)]
-
-
-def convolution_mnist(input_, trainable=True):
- with tf.variable_scope("conv_pool_1"):
- conv1 = conv_relu_pool(input_, [5, 5, 1, 20], [20], trainable=trainable)
- with tf.variable_scope("conv_pool_2"):
- conv2 = conv_relu_pool(conv1, [5, 5, 20, 50], [50], trainable=trainable)
- return conv2
-
-
-def fully_connected(conv_out):
- with tf.name_scope("fc_1"):
- init_dim = np.prod([s.value for s in conv_out.shape if s.value is not None])
- h_pool2_flat = tf.reshape(conv_out, [-1, init_dim])
- W_fc1 = weight_variable([init_dim, 4096*2])
- b_fc1 = bias_variable([4096*2])
- h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
- tf.summary.histogram("weights", W_fc1)
- tf.summary.histogram("biases", b_fc1)
-
- return h_fc1
+X_train, Y_train = mnist.train
+X_val, Y_val = mnist.validation
+X_test, Y_test = mnist.test
def tf_rbf_kernel(X, Y, gamma):
@@ -75,22 +37,32 @@ def tf_rbf_kernel(X, Y, gamma):
return K
-def nystrom_layer(input_x, input_subsample, gamma):
+def nystrom_layer(input_x, input_subsample, gamma, output_dim):
+ nystrom_sample_size = input_subsample.shape[0]
with tf.name_scope("nystrom"):
init_dim = np.prod([s.value for s in input_x.shape[1:] if s.value is not None])
h_conv_flat = tf.reshape(input_x, [-1, init_dim])
- h_conv_nystrom_subsample_flat = tf.reshape(input_subsample, [NYSTROM_SAMPLE_SIZE, init_dim])
+ h_conv_nystrom_subsample_flat = tf.reshape(input_subsample, [nystrom_sample_size, init_dim])
with tf.name_scope("kernel_vec"):
kernel_vector = tf_rbf_kernel(h_conv_flat, h_conv_nystrom_subsample_flat, gamma=gamma)
- D = weight_variable((NYSTROM_SAMPLE_SIZE,))
- V = weight_variable((NYSTROM_SAMPLE_SIZE, NYSTROM_SAMPLE_SIZE))
+ # this is the initial formulation given by sklearn
+ # D = tf.get_variable("D", [nystrom_sample_size,], initializer=tf.random_normal_initializer(stddev=0.1))
+ # V = tf.get_variable("V", [nystrom_sample_size, nystrom_sample_size],
+ # initializer=tf.random_normal_initializer(stddev=0.1))
+ # out_fc = tf.matmul(kernel_vector, tf.matmul(tf.multiply(D, V), tf.transpose(V)))
+
+ # this is simpler
+ W = tf.get_variable("W", [nystrom_sample_size, output_dim],
+ initializer=tf.random_normal_initializer(stddev=0.1))
+ out_fc = tf.matmul(kernel_vector, W)
- out_fc = tf.matmul(kernel_vector, tf.matmul(tf.multiply(D, V), tf.transpose(V)))
return out_fc
def main():
+ NYSTROM_SAMPLE_SIZE = 100
+ X_nystrom = X_train[np.random.permutation(NYSTROM_SAMPLE_SIZE)]
GAMMA = 0.001
print("Gamma = {}".format(GAMMA))
@@ -113,19 +85,9 @@ def main():
scope_conv_mnist.reuse_variables()
h_conv_nystrom_subsample = convolution_mnist(x_nystrom_image, trainable=False)
- out_fc = nystrom_layer(h_conv, h_conv_nystrom_subsample, GAMMA)
-
- # classification
- with tf.name_scope("fc_2"):
- keep_prob = tf.placeholder(tf.float32, name="keep_prob")
- h_fc1_drop = tf.nn.dropout(out_fc, keep_prob)
- dim = np.prod([s.value for s in h_fc1_drop.shape if s.value is not None])
- W_fc2 = weight_variable([dim, output_dim])
- b_fc2 = bias_variable([output_dim])
- tf.summary.histogram("weights", W_fc2)
- tf.summary.histogram("biases", b_fc2)
+ out_fc = nystrom_layer(h_conv, h_conv_nystrom_subsample, GAMMA, NYSTROM_SAMPLE_SIZE)
- y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
+ y_conv, keep_prob = classification_mnist(out_fc, output_dim=output_dim)
# # calcul de la loss
with tf.name_scope("xent"):
--
GitLab