Move some deepfriedconvnet repo script to skluc examples.

skluc/examples/deepfriedConvnetMnist.py

+"""
+Convolutional Neural Netwok implementation in tensorflow whith multiple representations possible after the convolution:
+    - Fully connected layer
+    - Random Fourier Features layer
+    - Fast Food layer where Fast Hadamard Transform has been replaced by dot product with Hadamard matrix.
+
+See:
+"Deep Fried Convnets" by
+Zichao Yang, Marcin Moczulski, Misha Denil, Nando de Freitas, Alex Smola, Le Song, Ziyu Wang
+
+"""
+
+import tensorflow as tf
+import numpy as np
+import skluc.data.mldatasets as dataset
+from skluc.tensorflow_.utils import convolution_mnist, classification_mnist, batch_generator
+from skluc.tensorflow_.kernel_approximation import fastfood_layer
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import time as t
+
+val_size = 5000
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.normalize()
+mnist.to_one_hot()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
+X_train, Y_train = mnist.train
+X_test, Y_test = mnist.test
+X_val, Y_val = mnist.validation
+
+
+if __name__ == '__main__':
+    SIGMA = 5.0
+    print("Sigma = {}".format(SIGMA))
+
+    with tf.Graph().as_default():
+        input_dim, output_dim = X_train.shape[1], Y_train.shape[1]
+
+        x = tf.placeholder(tf.float32, shape=[None, input_dim], name="x")
+        y_ = tf.placeholder(tf.float32, shape=[None, output_dim], name="labels")
+
+        # side size is width or height of the images
+        side_size = int(np.sqrt(input_dim))
+        x_image = tf.reshape(x, [-1, side_size, side_size, 1])
+        tf.summary.image("digit", x_image, max_outputs=3)
+
+        # Representation layer
+        h_conv = convolution_mnist(x_image)
+        # out_fc = fully_connected(h_conv)  # 95% accuracy
+        # out_fc = tf.nn.relu(fast_food(h_conv, SIGMA, nbr_stack=1))  # 83% accuracy (conv) | 56% accuracy (noconv)
+        # out_fc = tf.nn.relu(fast_food(h_conv, SIGMA, nbr_stack=2))
+        # out_fc = tf.nn.relu(fast_food(h_conv, SIGMA, nbr_stack=2, trainable=True))
+        # out_fc = tf.nn.relu(fast_food(h_conv, SIGMA, trainable=True))  # 84% accuracy (conv) | 59% accuracy (noconv)
+        out_fc = fastfood_layer(h_conv, SIGMA, nbr_stack=1, trainable=True)  # 84% accuracy (conv) | 59% accuracy (noconv)
+        # out_fc = random_features(h_conv, SIGMA)  # 82% accuracy (conv) | 47% accuracy (noconv)
+
+        # classification
+        y_conv, keep_prob = classification_mnist(out_fc, output_dim)
+
+        # calcul de la loss
+        with tf.name_scope("xent"):
+            cross_entropy = tf.reduce_mean(
+                tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv, name="xentropy"),
+                name="xentropy_mean")
+            tf.summary.scalar('loss-xent', cross_entropy)
+
+        # calcul du gradient
+        with tf.name_scope("train"):
+            global_step = tf.Variable(0, name="global_step", trainable=False)
+            train_optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy, global_step=global_step)
+
+        # calcul de l'accuracy
+        with tf.name_scope("accuracy"):
+            predictions = tf.argmax(y_conv, 1)
+            correct_prediction = tf.equal(predictions, tf.argmax(y_, 1))
+            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+            tf.summary.scalar("accuracy", accuracy)
+
+        merged_summary = tf.summary.merge_all()
+
+        init = tf.global_variables_initializer()
+        # Create a session for running Ops on the Graph.
+        sess = tf.Session()
+        # Instantiate a SummaryWriter to output summaries and the Graph.
+        summary_writer = tf.summary.FileWriter("results_deepfried_stacked")
+        summary_writer.add_graph(sess.graph)
+        # Initialize all Variable objects
+        sess.run(init)
+        # actual learning
+        started = t.time()
+        feed_dict_val = {x: X_val, y_: Y_val, keep_prob: 1.0}
+        for _ in range(1):
+            i = 0
+            for X_batch, Y_batch in batch_generator(X_train, Y_train, 64, circle=True):
+                feed_dict = {x: X_batch, y_: Y_batch, keep_prob: 0.5}
+                # le _ est pour capturer le retour de "train_optimizer" qu'il faut appeler
+                # pour calculer le gradient mais dont l'output ne nous interesse pas
+                _, loss, y_result, x_exp = sess.run([train_optimizer, cross_entropy, y_conv, x_image], feed_dict=feed_dict)
+                if i % 100 == 0:
+                    print('step {}, loss {} (with dropout)'.format(i, loss))
+                    r_accuracy = sess.run([accuracy], feed_dict=feed_dict_val)
+                    print("accuracy: {} on validation set (without dropout).".format(r_accuracy))
+                    summary_str = sess.run(merged_summary, feed_dict=feed_dict)
+                    summary_writer.add_summary(summary_str, i)
+                i += 1
+
+        stoped = t.time()
+        accuracy, preds = sess.run([accuracy, predictions], feed_dict={
+            x: X_test, y_: Y_test, keep_prob: 1.0})
+        print('test accuracy %g' % accuracy)
+        np.set_printoptions(threshold=np.nan)
+        print("Prediction sample: " + str(preds[:50]))
+        print("Actual values: " + str(np.argmax(Y_test[:50], axis=1)))
+        print("Elapsed time: %.4f s" % (stoped - started))
\ No newline at end of file

skluc/examples/time_batch_subsample_ops.py

+import tensorflow as tf
+import numpy as np
+from sklearn.metrics.pairwise import rbf_kernel
+
+import skluc.data.mldatasets as dataset
+from skluc.tensorflow_.utils import fully_connected, get_next_batch, tf_op, conv_relu_pool
+from skluc.utils import time_fct
+from skluc.tensorflow_.kernel import tf_rbf_kernel
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+# Preparing the dataset #########################
+
+mnist = dataset.MnistDataset()
+mnist.load()
+X_train, _ = mnist.train
+X_train = np.array(X_train / 255)
+X_train = X_train.astype(np.float32)
+
+################################################
+
+# todo timer les autres kernels pour verifier que c'est effectivement plus rapide
+
+if __name__ == '__main__':
+    input_dim = X_train.shape[1]
+    output_dim_fc = 4096*2
+    batch_size = 500
+    subsample_size = 500
+    X_batch = get_next_batch(X_train, 0, batch_size)
+    X_subsample = get_next_batch(X_train, 0, subsample_size)
+
+    with tf.Graph().as_default():
+        # inputs
+        x = tf.placeholder(tf.float32, shape=[None, input_dim], name="x")
+        x_subsample = tf.placeholder(tf.float32, shape=[None, input_dim], name="x_subsample")
+
+        # reshape vector inputs to images
+        side_size = int(np.sqrt(input_dim))
+        x_image = tf.reshape(x, [-1, side_size, side_size, 1])
+        x_subsample_image = tf.reshape(x_subsample, [subsample_size, side_size, side_size, 1])
+
+        # fully connected ops
+        out_fc_x = fully_connected(x, output_dim_fc, act=tf.nn.relu, variable_scope="fc_x")
+        out_fc_subsample = fully_connected(x_subsample, output_dim_fc, act=tf.nn.relu, variable_scope="fc_subsample")
+
+        # convolution ops
+        out_conv_x = conv_relu_pool(x_image, [5, 5, 1, 20], [20], pool_size=3, variable_scope="conv_x")
+        out_conv_subsample = conv_relu_pool(x_subsample_image, [5, 5, 1, 20], [20], pool_size=3,
+                                            variable_scope="conv_subsample")
+
+        init_dim = np.prod([s.value for s in out_conv_x.shape[1:] if s.value is not None])
+        x_conv_flat = tf.reshape(out_conv_x, [-1, init_dim])
+        subsample_conv_flat = tf.reshape(out_conv_subsample, [subsample_size, init_dim])
+
+        # kernel computing ops
+        with tf.device('/cpu:0'):
+            kernel_cpu = tf_rbf_kernel(x_conv_flat, subsample_conv_flat, gamma=0.001)
+        with tf.device('/device:GPU:0'):
+            kernel_gpu = tf_rbf_kernel(x_conv_flat, subsample_conv_flat, gamma=0.001)
+
+        feed_dict = {x: X_batch, x_subsample: X_subsample}
+
+        def kernel_sklearn():
+            with tf.Session() as sess:
+                init = tf.global_variables_initializer()
+                sess.run([init])
+                x, y = sess.run([x_conv_flat, subsample_conv_flat], feed_dict=feed_dict)
+            rbf_kernel(x, y, gamma=0.001)
+
+        d_time_results = {
+            "fc_x": lambda: time_fct(lambda: tf_op(feed_dict, [out_fc_x]), n_iter=10),
+            "fc_subsample": lambda: time_fct(lambda: tf_op(feed_dict, [out_fc_subsample]), n_iter=10),
+            "reshape_x": lambda: time_fct(lambda: tf_op(feed_dict, [x_image]), n_iter=10),
+            "reshape_subsample": lambda: time_fct(lambda: tf_op(feed_dict, [x_subsample_image]), n_iter=10),
+            "reshape_x + conv_x": lambda: time_fct(lambda: tf_op(feed_dict, [out_conv_x]), n_iter=10),
+            "reshape_subsample + conv_subsample": lambda: time_fct(lambda: tf_op(feed_dict, [out_conv_subsample]), n_iter=10),
+            "reshape_x + conv_x + reshape_subsample + conv_subsample": lambda: time_fct(lambda: tf_op(feed_dict, [out_conv_x, out_conv_subsample]), n_iter=10),
+            "reshape_x + conv_x + reshape_subsample + conv_subsample + kernel_cpu": lambda: time_fct(lambda: tf_op(feed_dict, [kernel_cpu]), n_iter=10),
+            "reshape_x + conv_x + reshape_subsample + conv_subsample + kernel_gpu": lambda: time_fct(lambda: tf_op(feed_dict, [kernel_gpu]), n_iter=10),
+            "reshape_x + conv_x + reshape_subsample + conv_subsample + kernel_sklearn": lambda: time_fct(kernel_sklearn, n_iter=10)
+        }
+
+        for key, value in d_time_results.items():
+            print("{}:\t{:.4f}s".format(key, value()))
+            tf.reset_default_graph()
+