usage of the new methods for preprocessing of datasets + usage of...

usage of the new methods for preprocessing of datasets + usage of batch_generator instead of get_next_batch

skluc/examples/fc_nn.py

@@ -8,37 +8,27 @@ where the input comes from memory.
 import tensorflow as tf
 import numpy as np
 import skluc.mldatasets as dataset
-from skluc.neural_networks import inference_mnist, get_next_batch
+from skluc.neural_networks import inference_mnist,  batch_generator
 
 tf.logging.set_verbosity(tf.logging.ERROR)
 
 import time as t
 
-from sklearn.preprocessing import LabelBinarizer
 
 # Preparing the dataset #########################
 
-enc = LabelBinarizer()
-mnist = dataset.MnistDataset()
-mnist = mnist.load()
-X_train, Y_train = mnist["train"]
-X_train = np.array(X_train / 255)
-enc.fit(Y_train)
-Y_train = np.array(enc.transform(Y_train))
-X_test, Y_test = mnist["test"]
-X_test = np.array(X_test / 255)
-Y_test = np.array(enc.transform(Y_test))
-
-X_train = X_train.astype(np.float32)
-permut = np.random.permutation(X_train.shape[0])
 val_size = 5000
-X_val = X_train[permut[:val_size]]
-X_train = X_train[permut[val_size:]]
-Y_val = Y_train[permut[:val_size]]
-Y_train = Y_train[permut[val_size:]]
-X_test = X_test.astype(np.float32)
-Y_train = Y_train.astype(np.float32)
-Y_test = Y_test.astype(np.float32)
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.to_one_hot()
+mnist.normalize()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
+
+X_train, Y_train = mnist.train
+X_val, Y_val = mnist.validation
+X_test, Y_test = mnist.test
+
 
 #################################################
 
@@ -92,9 +82,9 @@ def main():
         # actual learning
         started = t.time()
         feed_dict_val = {x: X_val, y_: Y_val, keep_prob: 1.0}
-        for i in range(2):
-            X_batch = get_next_batch(X_train, i, 64)
-            Y_batch = get_next_batch(Y_train, i, 64)
+        for _ in range(1):
+            i = 0
+            for X_batch, Y_batch in batch_generator(X_train, Y_train, 64, circle=True):
                 feed_dict = {x: X_batch, y_: Y_batch, keep_prob: 0.5}
                 # le _ est pour capturer le retour de "train_optimizer" qu'il faut appeler
                 # pour calculer le gradient mais dont l'output ne nous interesse pas
@@ -105,6 +95,7 @@ def main():
                     print("accuracy: {} on validation set (without dropout).".format(r_accuracy))
                     summary_str = sess.run(merged_summary, feed_dict=feed_dict)
                     summary_writer.add_summary(summary_str, i)
+                i += 1
 
         stoped = t.time()
         accuracy, preds = sess.run([accuracy, predictions], feed_dict={

skluc/examples/fc_nn_timed.py

@@ -9,10 +9,12 @@ tf.logging.set_verbosity(tf.logging.ERROR)
 # Preparing the dataset #########################
 
 mnist = dataset.MnistDataset()
-mnist = mnist.load()
-X_train, _ = mnist["train"]
-X_train = np.array(X_train / 255)
-X_train = X_train.astype(np.float32)
+mnist.load()
+mnist.normalize()
+mnist.data_astype(np.float32)
+
+X_train, _ = mnist.train
+
 
 #################################################
 

skluc/examples/tfrecord_nn.py

@@ -18,33 +18,18 @@ tf.logging.set_verbosity(tf.logging.ERROR)
 import time as t
 from collections import namedtuple
 
-dataset_mnist = dataset.MnistDataset()
-dataset_mnist.load()
-mnist = dataset_mnist.to_image()
-
-X_train, Y_train = mnist["train"]
-X_test, Y_test = mnist["test"]
-
-# normalize
-X_train = np.array(X_train / 255)
-X_test = np.array(X_test / 255)
-X_train = X_train.astype(np.float32)
-Y_train = Y_train.astype(np.float32)
-
-# define validation
-permut = np.random.permutation(X_train.shape[0])
 val_size = 5000
-
-X_val = X_train[permut[:val_size]]
-Y_val = Y_train[permut[:val_size]]
-Y_val = Y_val.astype(np.float32)
-
-X_train = X_train[permut[val_size:]]
-Y_train = Y_train[permut[val_size:]]
-
-# Define test set
-X_test = X_test.astype(np.float32)
-Y_test = Y_test.astype(np.float32)
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.to_one_hot()
+mnist.to_image()
+mnist.normalize()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
+
+X_train, Y_train = mnist.train
+X_val, Y_val = mnist.validation
+X_test, Y_test = mnist.test
 
 # build dataset objects
 
@@ -193,6 +178,7 @@ def trainning():
             keep_prob_train: 0.5,
             keep_prob_test: 1.0
         }
+        # todo maybe there is a misunderstanding of epoch definition here.
         for i in range(num_epochs):
             # run training and get the loss
             _, loss = sess.run([train_optimizer, cross_entropy], feed_dict=feed_dict)

skluc/examples/write_read_tfrecords.py

@@ -13,25 +13,19 @@ tf.logging.set_verbosity(tf.logging.ERROR)
 
 from collections import namedtuple
 
-dataset_mnist = dataset.MnistDataset()
-dataset_mnist.load()
-mnist = dataset_mnist.to_image()
-
-X_train, Y_train = mnist["train"]
-X_train = np.array(X_train / 255)
-X_test, Y_test = mnist["test"]
-X_test = np.array(X_test / 255)
-
-X_train = X_train.astype(np.float32)
-permut = np.random.permutation(X_train.shape[0])
 val_size = 5000
-X_val = X_train[permut[:val_size]]
-X_train = X_train[permut[val_size:]]
-Y_val = Y_train[permut[:val_size]]
-Y_train = Y_train[permut[val_size:]]
-X_test = X_test.astype(np.float32)
-Y_train = Y_train.astype(np.float32)
-Y_test = Y_test.astype(np.float32)
+mnist = dataset.MnistDataset(validation_size=val_size)
+mnist.load()
+mnist.to_one_hot()
+mnist.normalize()
+mnist.to_image()
+mnist.data_astype(np.float32)
+mnist.labels_astype(np.float32)
+
+X_train, Y_train = mnist.train
+X_val, Y_val = mnist.validation
+X_test, Y_test = mnist.test
+
 
 train = namedtuple("Dataset", ["images", "labels", "num_examples"])
 train.images = X_train