change in the dataset preprocessing + the nystrom layer has now only one...

change in the dataset preprocessing + the nystrom layer has now only one matrix W instead of D and V

main/nystrom/nystrom_approx.py

@@ -2,65 +2,27 @@
 Convnet with nystrom approximation of the feature map.
 
 """
+import time as t
 
 import tensorflow as tf
 import numpy as np
 
 import skluc.mldatasets as dataset
-from skluc.neural_networks import bias_variable, weight_variable, conv_relu_pool, get_next_batch
+from skluc.neural_networks import get_next_batch, classification_mnist, convolution_mnist
 
 tf.logging.set_verbosity(tf.logging.ERROR)
 
-import time as t
-
-from sklearn.preprocessing import LabelBinarizer
-
-enc = LabelBinarizer()
-mnist = dataset.MnistDataset()
-mnist = mnist.load()
-X_train, Y_train = mnist["train"]
-X_train = np.array(X_train / 255)
-enc.fit(Y_train)
-Y_train = np.array(enc.transform(Y_train))
-X_test, Y_test = mnist["test"]
-X_test = np.array(X_test / 255)
-Y_test = np.array(enc.transform(Y_test))
-
-X_train = X_train.astype(np.float32)
-permut = np.random.permutation(X_train.shape[0])
 val_size = 5000
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.to_one_hot()
+mnist.normalize()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
 
-X_val = X_train[permut[:val_size]]
-Y_val = Y_train[permut[:val_size]]
-X_train = X_train[permut[val_size:]]
-Y_train = Y_train[permut[val_size:]]
-X_test = X_test.astype(np.float32)
-Y_train = Y_train.astype(np.float32)
-Y_test = Y_test.astype(np.float32)
-
-NYSTROM_SAMPLE_SIZE = 500
-X_nystrom = X_train[np.random.permutation(NYSTROM_SAMPLE_SIZE)]
-
-
-def convolution_mnist(input_, trainable=True):
-    with tf.variable_scope("conv_pool_1"):
-        conv1 = conv_relu_pool(input_, [5, 5, 1, 20], [20], trainable=trainable)
-    with tf.variable_scope("conv_pool_2"):
-        conv2 = conv_relu_pool(conv1, [5, 5, 20, 50], [50], trainable=trainable)
-    return conv2
-
-
-def fully_connected(conv_out):
-    with tf.name_scope("fc_1"):
-        init_dim = np.prod([s.value for s in conv_out.shape if s.value is not None])
-        h_pool2_flat = tf.reshape(conv_out, [-1, init_dim])
-        W_fc1 = weight_variable([init_dim, 4096*2])
-        b_fc1 = bias_variable([4096*2])
-        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
-        tf.summary.histogram("weights", W_fc1)
-        tf.summary.histogram("biases", b_fc1)
-
-    return h_fc1
+X_train, Y_train = mnist.train
+X_val, Y_val = mnist.validation
+X_test, Y_test = mnist.test
 
 
 def tf_rbf_kernel(X, Y, gamma):
@@ -75,22 +37,32 @@ def tf_rbf_kernel(X, Y, gamma):
     return K
 
 
-def nystrom_layer(input_x, input_subsample, gamma):
+def nystrom_layer(input_x, input_subsample, gamma, output_dim):
+    nystrom_sample_size = input_subsample.shape[0]
     with tf.name_scope("nystrom"):
         init_dim = np.prod([s.value for s in input_x.shape[1:] if s.value is not None])
         h_conv_flat = tf.reshape(input_x, [-1, init_dim])
-        h_conv_nystrom_subsample_flat = tf.reshape(input_subsample, [NYSTROM_SAMPLE_SIZE, init_dim])
+        h_conv_nystrom_subsample_flat = tf.reshape(input_subsample, [nystrom_sample_size, init_dim])
         with tf.name_scope("kernel_vec"):
             kernel_vector = tf_rbf_kernel(h_conv_flat, h_conv_nystrom_subsample_flat, gamma=gamma)
 
-        D = weight_variable((NYSTROM_SAMPLE_SIZE,))
-        V = weight_variable((NYSTROM_SAMPLE_SIZE, NYSTROM_SAMPLE_SIZE))
+        # this is the initial formulation given by sklearn
+        # D = tf.get_variable("D", [nystrom_sample_size,], initializer=tf.random_normal_initializer(stddev=0.1))
+        # V = tf.get_variable("V", [nystrom_sample_size, nystrom_sample_size],
+        # initializer=tf.random_normal_initializer(stddev=0.1))
+        # out_fc = tf.matmul(kernel_vector, tf.matmul(tf.multiply(D, V), tf.transpose(V)))
+
+        # this is simpler
+        W = tf.get_variable("W", [nystrom_sample_size, output_dim],
+                            initializer=tf.random_normal_initializer(stddev=0.1))
+        out_fc = tf.matmul(kernel_vector, W)
 
-        out_fc = tf.matmul(kernel_vector, tf.matmul(tf.multiply(D, V), tf.transpose(V)))
     return out_fc
 
 
 def main():
+    NYSTROM_SAMPLE_SIZE = 100
+    X_nystrom = X_train[np.random.permutation(NYSTROM_SAMPLE_SIZE)]
     GAMMA = 0.001
     print("Gamma = {}".format(GAMMA))
 
@@ -113,19 +85,9 @@ def main():
             scope_conv_mnist.reuse_variables()
             h_conv_nystrom_subsample = convolution_mnist(x_nystrom_image, trainable=False)
 
-        out_fc = nystrom_layer(h_conv, h_conv_nystrom_subsample, GAMMA)
-
-        # classification
-        with tf.name_scope("fc_2"):
-            keep_prob = tf.placeholder(tf.float32, name="keep_prob")
-            h_fc1_drop = tf.nn.dropout(out_fc, keep_prob)
-            dim = np.prod([s.value for s in h_fc1_drop.shape if s.value is not None])
-            W_fc2 = weight_variable([dim, output_dim])
-            b_fc2 = bias_variable([output_dim])
-            tf.summary.histogram("weights", W_fc2)
-            tf.summary.histogram("biases", b_fc2)
+        out_fc = nystrom_layer(h_conv, h_conv_nystrom_subsample, GAMMA, NYSTROM_SAMPLE_SIZE)
 
-            y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
+        y_conv, keep_prob = classification_mnist(out_fc, output_dim=output_dim)
 
         # # calcul de la loss
         with tf.name_scope("xent"):