Skip to content
Snippets Groups Projects
Commit 0df9fb21 authored by Luc Giffon's avatar Luc Giffon
Browse files

usage of the new methods for preprocessing of datasets + usage of...

usage of the new methods for preprocessing of datasets + usage of batch_generator instead of get_next_batch
parent 80fdae53
No related branches found
No related tags found
No related merge requests found
......@@ -8,37 +8,27 @@ where the input comes from memory.
import tensorflow as tf
import numpy as np
import skluc.mldatasets as dataset
from skluc.neural_networks import inference_mnist, get_next_batch
from skluc.neural_networks import inference_mnist, batch_generator
tf.logging.set_verbosity(tf.logging.ERROR)
import time as t
from sklearn.preprocessing import LabelBinarizer
# Preparing the dataset #########################
enc = LabelBinarizer()
mnist = dataset.MnistDataset()
mnist = mnist.load()
X_train, Y_train = mnist["train"]
X_train = np.array(X_train / 255)
enc.fit(Y_train)
Y_train = np.array(enc.transform(Y_train))
X_test, Y_test = mnist["test"]
X_test = np.array(X_test / 255)
Y_test = np.array(enc.transform(Y_test))
X_train = X_train.astype(np.float32)
permut = np.random.permutation(X_train.shape[0])
val_size = 5000
X_val = X_train[permut[:val_size]]
X_train = X_train[permut[val_size:]]
Y_val = Y_train[permut[:val_size]]
Y_train = Y_train[permut[val_size:]]
X_test = X_test.astype(np.float32)
Y_train = Y_train.astype(np.float32)
Y_test = Y_test.astype(np.float32)
mnist = dataset.MnistDataset(validation_size=val_size)
mnist.load()
mnist.to_one_hot()
mnist.normalize()
mnist.data_astype(np.float32)
mnist.labels_astype(np.float32)
X_train, Y_train = mnist.train
X_val, Y_val = mnist.validation
X_test, Y_test = mnist.test
#################################################
......@@ -92,9 +82,9 @@ def main():
# actual learning
started = t.time()
feed_dict_val = {x: X_val, y_: Y_val, keep_prob: 1.0}
for i in range(2):
X_batch = get_next_batch(X_train, i, 64)
Y_batch = get_next_batch(Y_train, i, 64)
for _ in range(1):
i = 0
for X_batch, Y_batch in batch_generator(X_train, Y_train, 64, circle=True):
feed_dict = {x: X_batch, y_: Y_batch, keep_prob: 0.5}
# le _ est pour capturer le retour de "train_optimizer" qu'il faut appeler
# pour calculer le gradient mais dont l'output ne nous interesse pas
......@@ -105,6 +95,7 @@ def main():
print("accuracy: {} on validation set (without dropout).".format(r_accuracy))
summary_str = sess.run(merged_summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, i)
i += 1
stoped = t.time()
accuracy, preds = sess.run([accuracy, predictions], feed_dict={
......
......@@ -9,10 +9,12 @@ tf.logging.set_verbosity(tf.logging.ERROR)
# Preparing the dataset #########################
mnist = dataset.MnistDataset()
mnist = mnist.load()
X_train, _ = mnist["train"]
X_train = np.array(X_train / 255)
X_train = X_train.astype(np.float32)
mnist.load()
mnist.normalize()
mnist.data_astype(np.float32)
X_train, _ = mnist.train
#################################################
......
......@@ -18,33 +18,18 @@ tf.logging.set_verbosity(tf.logging.ERROR)
import time as t
from collections import namedtuple
dataset_mnist = dataset.MnistDataset()
dataset_mnist.load()
mnist = dataset_mnist.to_image()
X_train, Y_train = mnist["train"]
X_test, Y_test = mnist["test"]
# normalize
X_train = np.array(X_train / 255)
X_test = np.array(X_test / 255)
X_train = X_train.astype(np.float32)
Y_train = Y_train.astype(np.float32)
# define validation
permut = np.random.permutation(X_train.shape[0])
val_size = 5000
X_val = X_train[permut[:val_size]]
Y_val = Y_train[permut[:val_size]]
Y_val = Y_val.astype(np.float32)
X_train = X_train[permut[val_size:]]
Y_train = Y_train[permut[val_size:]]
# Define test set
X_test = X_test.astype(np.float32)
Y_test = Y_test.astype(np.float32)
mnist = dataset.MnistDataset(validation_size=val_size)
mnist.load()
mnist.to_one_hot()
mnist.to_image()
mnist.normalize()
mnist.data_astype(np.float32)
mnist.labels_astype(np.float32)
X_train, Y_train = mnist.train
X_val, Y_val = mnist.validation
X_test, Y_test = mnist.test
# build dataset objects
......@@ -193,6 +178,7 @@ def trainning():
keep_prob_train: 0.5,
keep_prob_test: 1.0
}
# todo maybe there is a misunderstanding of epoch definition here.
for i in range(num_epochs):
# run training and get the loss
_, loss = sess.run([train_optimizer, cross_entropy], feed_dict=feed_dict)
......
......@@ -13,25 +13,19 @@ tf.logging.set_verbosity(tf.logging.ERROR)
from collections import namedtuple
dataset_mnist = dataset.MnistDataset()
dataset_mnist.load()
mnist = dataset_mnist.to_image()
X_train, Y_train = mnist["train"]
X_train = np.array(X_train / 255)
X_test, Y_test = mnist["test"]
X_test = np.array(X_test / 255)
X_train = X_train.astype(np.float32)
permut = np.random.permutation(X_train.shape[0])
val_size = 5000
X_val = X_train[permut[:val_size]]
X_train = X_train[permut[val_size:]]
Y_val = Y_train[permut[:val_size]]
Y_train = Y_train[permut[val_size:]]
X_test = X_test.astype(np.float32)
Y_train = Y_train.astype(np.float32)
Y_test = Y_test.astype(np.float32)
mnist = dataset.MnistDataset(validation_size=val_size)
mnist.load()
mnist.to_one_hot()
mnist.normalize()
mnist.to_image()
mnist.data_astype(np.float32)
mnist.labels_astype(np.float32)
X_train, Y_train = mnist.train
X_val, Y_val = mnist.validation
X_test, Y_test = mnist.test
train = namedtuple("Dataset", ["images", "labels", "num_examples"])
train.images = X_train
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment