example directory with example of nn with memory inpu + example of nn with...

example directory with example of nn with memory inpu + example of nn with tfrecords input + usage example of read and write tfrecords

skluc/examples/fc_nn.py

+"""
+Convolutional Neural Netwok implementation in tensorflow whith fully connected layers.
+
+The neural network is ran against the mnist dataset and we can see an example of distortion of input in the case
+where the input comes from memory.
+"""
+
+import tensorflow as tf
+import numpy as np
+import skluc.mldatasets as dataset
+from skluc.neural_networks import bias_variable, weight_variable, conv2d, max_pool_2x2
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import time as t
+
+from sklearn.preprocessing import LabelBinarizer
+
+# Preparing the dataset #########################
+
+enc = LabelBinarizer()
+mnist = dataset.MnistDataset()
+mnist = mnist.load()
+X_train, Y_train = mnist["train"]
+X_train = np.array(X_train / 255)
+enc.fit(Y_train)
+Y_train = np.array(enc.transform(Y_train))
+X_test, Y_test = mnist["test"]
+X_test = np.array(X_test / 255)
+Y_test = np.array(enc.transform(Y_test))
+
+X_train = X_train.astype(np.float32)
+permut = np.random.permutation(X_train.shape[0])
+val_size = 5000
+X_val = X_train[permut[:val_size]]
+X_train = X_train[permut[val_size:]]
+Y_val = Y_train[permut[:val_size]]
+Y_train = Y_train[permut[val_size:]]
+X_test = X_test.astype(np.float32)
+Y_train = Y_train.astype(np.float32)
+Y_test = Y_test.astype(np.float32)
+
+#################################################
+
+
+def convolution_mnist(input):
+    with tf.name_scope("conv_pool_1"):
+        # 32 is the number of filter we'll use. e.g. the number of different
+        # shapes this layer is able to recognize
+        W_conv1 = weight_variable([5, 5, 1, 20])
+        tf.summary.histogram("weights conv1", W_conv1)
+        b_conv1 = bias_variable([20])
+        tf.summary.histogram("biases conv1", b_conv1)
+        # -1 is here to keep the total size constant (784)
+        h_conv1 = tf.nn.relu(conv2d(input, W_conv1) + b_conv1)
+        tf.summary.histogram("act conv1", h_conv1)
+        h_pool1 = max_pool_2x2(h_conv1)
+
+    with tf.name_scope("conv_pool_2"):
+        W_conv2 = weight_variable([5, 5, 20, 50])
+        tf.summary.histogram("weights conv2", W_conv2)
+        b_conv2 = bias_variable([50])
+        tf.summary.histogram("biases conv2", b_conv2)
+        h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
+        tf.summary.histogram("act conv2", h_conv2)
+        h_pool2 = max_pool_2x2(h_conv2)
+
+    return h_pool2
+
+
+def random_variable(shape, sigma):
+    W = np.random.normal(size=shape, scale=sigma).astype(np.float32)
+    return tf.Variable(W, name="random_Weights", trainable=False)
+
+
+def random_biases(shape):
+    b = np.random.uniform(0, 2 * np.pi, size=shape).astype(np.float32)
+    return tf.Variable(b, name="random_biase", trainable=False)
+
+
+def fully_connected(conv_out):
+    with tf.name_scope("fc_1"):
+        init_dim = np.prod([s.value for s in conv_out.shape if s.value is not None])
+        h_pool2_flat = tf.reshape(conv_out, [-1, init_dim])
+        W_fc1 = weight_variable([init_dim, 4096*2])
+        b_fc1 = bias_variable([4096*2])
+        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
+        tf.summary.histogram("weights", W_fc1)
+        tf.summary.histogram("biases", b_fc1)
+
+    return h_fc1
+
+
+def get_next_batch(full_set, batch_nbr, batch_size):
+    """
+    Return the next batch of a dataset.
+
+    This function assumes that all the previous batches of this dataset have been taken with the same size.
+
+    :param full_set: the full dataset from which the batch will be taken
+    :param batch_nbr: the number of the batch
+    :param batch_size: the size of the batch
+    :return:
+    """
+    index_start = (batch_nbr * batch_size) % full_set.shape[0]
+    index_stop = ((batch_nbr + 1) * batch_size) % full_set.shape[0]
+    if index_stop > index_start:
+        return full_set[index_start:index_stop]
+    else:
+        part1 = full_set[index_start:]
+        part2 = full_set[:index_stop]
+        return np.vstack((part1, part2))
+
+
+if __name__ == '__main__':
+    SIGMA = 5.0
+    print("Sigma = {}".format(SIGMA))
+
+    with tf.Graph().as_default():
+        input_dim, output_dim = X_train.shape[1], Y_train.shape[1]
+
+        x = tf.placeholder(tf.float32, shape=[None, input_dim], name="x")
+        y_ = tf.placeholder(tf.float32, shape=[None, output_dim], name="labels")
+
+        # side size is width or height of the images
+        side_size = int(np.sqrt(input_dim))
+        x_image = tf.reshape(x, [-1, side_size, side_size, 1])
+        tf.summary.image("digit", x_image, max_outputs=3)
+        # this is how we apply distortion but it is not used afterward
+        x_image_distorded = tf.image.random_brightness(x_image, max_delta=30)
+        tf.summary.image("digit_distorded", x_image_distorded, max_outputs=3)
+
+        # Representation layer
+        h_conv = convolution_mnist(x_image)
+        out_fc = fully_connected(h_conv)  # 95% accuracy
+
+        # classification
+        with tf.name_scope("fc_2"):
+            keep_prob = tf.placeholder(tf.float32, name="keep_prob")
+            h_fc1_drop = tf.nn.dropout(out_fc, keep_prob)
+            dim = np.prod([s.value for s in h_fc1_drop.shape if s.value is not None])
+            W_fc2 = weight_variable([dim, output_dim])
+            b_fc2 = bias_variable([output_dim])
+            tf.summary.histogram("weights", W_fc2)
+            tf.summary.histogram("biases", b_fc2)
+
+            y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
+
+        # calcul de la loss
+        with tf.name_scope("xent"):
+            cross_entropy = tf.reduce_mean(
+                tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv, name="xentropy"),
+                name="xentropy_mean")
+            tf.summary.scalar('loss-xent', cross_entropy)
+
+        # calcul du gradient
+        with tf.name_scope("train"):
+            global_step = tf.Variable(0, name="global_step", trainable=False)
+            train_optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy, global_step=global_step)
+
+        # calcul de l'accuracy
+        with tf.name_scope("accuracy"):
+            predictions = tf.argmax(y_conv, 1)
+            correct_prediction = tf.equal(predictions, tf.argmax(y_, 1))
+            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+            tf.summary.scalar("accuracy", accuracy)
+
+        merged_summary = tf.summary.merge_all()
+
+        init = tf.global_variables_initializer()
+        # Create a session for running Ops on the Graph.
+        sess = tf.Session()
+        # Instantiate a SummaryWriter to output summaries and the Graph.
+        summary_writer = tf.summary.FileWriter("results_fc_distorded")
+        summary_writer.add_graph(sess.graph)
+        # Initialize all Variable objects
+        sess.run(init)
+        # actual learning
+        started = t.time()
+        feed_dict_val = {x: X_val, y_: Y_val, keep_prob: 1.0}
+        for i in range(2):
+            X_batch = get_next_batch(X_train, i, 64)
+            Y_batch = get_next_batch(Y_train, i, 64)
+            feed_dict = {x: X_batch, y_: Y_batch, keep_prob: 0.5}
+            # le _ est pour capturer le retour de "train_optimizer" qu'il faut appeler
+            # pour calculer le gradient mais dont l'output ne nous interesse pas
+            _, loss = sess.run([train_optimizer, cross_entropy], feed_dict=feed_dict)
+            if i % 100 == 0:
+                print('step {}, loss {} (with dropout)'.format(i, loss))
+                r_accuracy = sess.run([accuracy], feed_dict=feed_dict_val)
+                print("accuracy: {} on validation set (without dropout).".format(r_accuracy))
+                summary_str = sess.run(merged_summary, feed_dict=feed_dict)
+                summary_writer.add_summary(summary_str, i)
+
+        stoped = t.time()
+        accuracy, preds = sess.run([accuracy, predictions], feed_dict={
+            x: X_test, y_: Y_test, keep_prob: 1.0})
+        print('test accuracy %g' % accuracy)
+        np.set_printoptions(threshold=np.nan)
+        print("Prediction sample: " + str(preds[:50]))
+        print("Actual values: " + str(np.argmax(Y_test[:50], axis=1)))
+        print("Elapsed time: %.4f s" % (stoped - started))
\ No newline at end of file

skluc/examples/tfrecord_nn.py

+"""
+Convolutional Neural Netwok implementation in tensorflow using tfrecords as inputs.
+
+From memory, the tfrecords ar written then read.
+
+The neural network is ran against the mnist dataset and we can see an example of distortion of input in
+the case of tfrecords data source.
+"""
+
+import tensorflow as tf
+import numpy as np
+import skluc.mldatasets as dataset
+from skluc.convert_image_to_records import convert_to
+from skluc.neural_networks import conv2d, max_pool_2x2
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+import time as t
+from collections import namedtuple
+
+dataset_mnist = dataset.MnistDataset()
+dataset_mnist.load()
+mnist = dataset_mnist.to_image()
+
+X_train, Y_train = mnist["train"]
+X_test, Y_test = mnist["test"]
+
+# normalize
+X_train = np.array(X_train / 255)
+X_test = np.array(X_test / 255)
+X_train = X_train.astype(np.float32)
+Y_train = Y_train.astype(np.float32)
+
+# define validation
+permut = np.random.permutation(X_train.shape[0])
+val_size = 5000
+
+X_val = X_train[permut[:val_size]]
+Y_val = Y_train[permut[:val_size]]
+Y_val = Y_val.astype(np.float32)
+
+X_train = X_train[permut[val_size:]]
+Y_train = Y_train[permut[val_size:]]
+
+# Define test set
+X_test = X_test.astype(np.float32)
+Y_test = Y_test.astype(np.float32)
+
+# build dataset objects
+
+train = namedtuple("Dataset", ["images", "labels", "num_examples"])
+train.images = X_train
+train.labels = Y_train
+train.num_examples = X_train.shape[0]
+
+test = namedtuple("Dataset", ["images", "labels", "num_examples"])
+test.images = X_test
+test.labels = Y_test
+test.num_examples = X_test.shape[0]
+
+val = namedtuple("Dataset", ["images", "labels", "num_examples"])
+val.images = X_val
+val.labels = Y_val
+val.num_examples = X_val.shape[0]
+
+
+def conv_relu_pool(input_, kernel_shape, bias_shape):
+    """
+    Generic function for defining a convolutional layer.
+
+    :param input_: The input tensor to be convoluted
+    :param kernel_shape: The shape of the kernels/filters
+    :param bias_shape: The shape of the bias
+    :return: The output tensor of the convolution
+    """
+    weights = tf.get_variable("weights", kernel_shape, initializer=tf.random_normal_initializer(stddev=0.1))
+    biases = tf.get_variable("biases", bias_shape, initializer=tf.constant_initializer(0.0))
+    tf.summary.histogram("weights", weights)
+    tf.summary.histogram("biases", biases)
+    conv = conv2d(input_, weights)
+    relu = tf.nn.relu(conv + biases)
+    tf.summary.histogram("act", relu)
+    pool = max_pool_2x2(relu)
+    return pool
+
+
+def fully_connected(input_, output_dim, act=tf.nn.relu):
+    """
+    Build a fully connected layer using input_ as input and output_dim as output size.
+
+    If the input_ is not the correct shape, it is reshaped.
+    """
+    input_dim = np.prod([s.value for s in input_.shape if s.value is not None])
+    if len(input_.shape) != 2:
+        print("[WARNING] {} input of fully_connected function has shape size > 2, e.g. {}. Reshaping."
+              .format(input_.name, len(input_.shape)))
+        input_flat = tf.reshape(input_, [-1, input_dim])
+    else:
+        input_flat = input_
+    weights = tf.get_variable("weights", [input_dim, output_dim], initializer=tf.random_normal_initializer(stddev=0.1))
+    biases = tf.get_variable("biases", [output_dim], initializer=tf.constant_initializer(0.0))
+    fc = tf.matmul(input_flat, weights) + biases
+    if act is not None:
+        fc = tf.nn.relu(fc)
+    tf.summary.histogram("weights", weights)
+    tf.summary.histogram("biases", biases)
+    return fc
+
+
+def convolution_mnist(input_):
+    """
+    Define the two convolutionnal layers used for mnist.
+
+    :param input_: The input images to be convoluted
+    :return:
+    """
+    with tf.variable_scope("conv_pool_1"):
+        conv1 = conv_relu_pool(input_, [5, 5, 1, 20], [20])
+    with tf.variable_scope("conv_pool_2"):
+        conv2 = conv_relu_pool(conv1, [5, 5, 20, 50], [50])
+    return conv2
+
+
+def fc_mnist(input_):
+    """
+    Define the fully connected layers used for mnist.
+
+    :param input_: The input tensor
+    :return:
+    """
+    with tf.variable_scope("fc_relu_1"):
+        fc = fully_connected(input_, 4096*2)
+    return fc
+
+
+def classification_mnist(input_, output_dim):
+    """
+    Define the classification layer used for mnist
+
+    :param input_: The input to classify
+    :param output_dim: The returned output dimension
+    :return:
+    """
+    with tf.variable_scope("fc_relu_2"):
+        keep_prob = tf.placeholder(tf.float32, name="keep_prob")
+        input_drop = tf.nn.dropout(input_, keep_prob)
+        y_ = fully_connected(input_drop, output_dim)
+    return y_, keep_prob
+
+
+def decode(serialized_example):
+    features = tf.parse_single_example(
+        serialized_example,
+        features={
+            'image_raw': tf.FixedLenFeature([], tf.string),
+            'label': tf.FixedLenFeature([], tf.int64),
+            'height': tf.FixedLenFeature([], tf.int64),
+            'width': tf.FixedLenFeature([], tf.int64),
+            'depth': tf.FixedLenFeature([], tf.int64)
+        }
+    )
+    image = tf.decode_raw(features['image_raw'], tf.float32)
+    image.set_shape((784))
+
+    label = tf.cast(features['label'], tf.int32)
+    label = tf.one_hot(label, 10)
+
+    return image, label
+
+
+def distortion(image, label):
+    """
+    Apply som distortion to the input images
+
+    :param image:
+    :param label:
+    :return:
+    """
+    distorted_image = tf.image.random_brightness(image,
+                                                 max_delta=15)
+    return distorted_image, label
+
+
+def get_tf_record(record_filename, num_epochs, batch_size, distord=True):
+    """
+    Gives an iteror to plug to the input of the neural network.
+
+    The iterator gives a new bach on demand.
+
+    :param record_filename: The filename where to find tfrecords.
+    :param num_epochs: The number of epoch we will need
+    :param batch_size: The size of each returned batch
+    :param distord: Parameter for applying or not distortion on input
+    :return: The iterator to plug to the input of the network
+    """
+    dataset = tf.data.TFRecordDataset(record_filename)
+    dataset = dataset.repeat(num_epochs)
+    dataset = dataset.map(decode)
+    # if distord:
+    #     dataset = dataset.map(distortion)
+    dataset = dataset.shuffle(1000 + 3 * batch_size)
+    dataset = dataset.batch(batch_size)  # combine les éléments en batchs
+    iterator = dataset.make_one_shot_iterator()
+    return iterator
+
+
+def inference_mnist(x_image, output_dim):
+    """
+    Compute inference on the class of the given tensor images.
+
+    :param x_image: Input tensor images
+    :param output_dim: The number of class
+    :return: The predicted classes
+    """
+    h_conv = convolution_mnist(x_image)
+    out_fc = fc_mnist(h_conv)
+    y_out, keep_prob = classification_mnist(out_fc, output_dim)
+    return y_out, keep_prob
+
+
+def train():
+    SIGMA = 5.0
+    num_epochs = 10000
+    batch_size = 64
+    print("Sigma = {}".format(SIGMA))
+
+    with tf.Graph().as_default():
+        # retourne batchs après batchs
+        iterator_train = get_tf_record("/tmp/data/mnist/mnist_train.tfrecords", num_epochs, batch_size, distord=False)
+        iterator_val = get_tf_record("/tmp/data/mnist/mnist_val.tfrecords", num_epochs, batch_size)
+        iterator_test = get_tf_record("/tmp/data/mnist/mnist_test.tfrecords", num_epochs, batch_size)
+
+        input_dim, output_dim = 784, 10
+
+        with tf.name_scope("train_set"):
+            x_image_train, y_train = iterator_train.get_next()
+            x_image_train = tf.reshape(x_image_train, [-1, 28, 28, 1])
+            tf.summary.image("digit", x_image_train, max_outputs=3)
+
+        with tf.name_scope("val_set"):
+            x_image_val, y_val = iterator_val.get_next()
+            x_image_val = tf.reshape(x_image_val, tf.stack([-1, 28, 28, 1]))
+
+        with tf.name_scope("test_set"):
+            x_image_test, y_test = iterator_test.get_next()
+            x_image_test = tf.reshape(x_image_test, tf.stack([-1, 28, 28, 1]))
+
+        with tf.variable_scope("inference") as scope_inference:
+            y_train_out, keep_prob_train = inference_mnist(x_image_train, output_dim)
+            scope_inference.reuse_variables()
+            y_val_out, keep_prob_val = inference_mnist(x_image_val, output_dim)
+            scope_inference.reuse_variables()
+            y_test_out, keep_prob_test = inference_mnist(x_image_test, output_dim)
+
+        # calcul de la loss
+        with tf.name_scope("xent"):
+            cross_entropy = tf.reduce_mean(
+                tf.nn.softmax_cross_entropy_with_logits(labels=y_train, logits=y_train_out, name="xentropy"),
+                name="xentropy_mean")
+            tf.summary.scalar('loss-xent', cross_entropy)
+
+        # calcul du gradient
+        with tf.name_scope("train"):
+            global_step = tf.Variable(0, name="global_step", trainable=False)
+            train_optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(cross_entropy, global_step=global_step)
+
+        # calcul de l'accuracy
+        with tf.name_scope("accuracy"):
+            predictions_val = tf.argmax(y_val_out, 1)
+            correct_prediction_val = tf.equal(predictions_val, tf.argmax(y_val, 1))
+            accuracy_val = tf.reduce_mean(tf.cast(correct_prediction_val, tf.float32))
+            tf.summary.scalar("accuracy", accuracy_val)
+            predictions_test = tf.argmax(y_test_out, 1)
+            correct_prediction_test = tf.equal(predictions_test, tf.argmax(y_test, 1))
+            accuracy_test = tf.reduce_mean(tf.cast(correct_prediction_test, tf.float32))
+
+        merged_summary = tf.summary.merge_all()
+
+        init = tf.global_variables_initializer()
+        # Create a session for running Ops on the Graph.
+        sess = tf.Session()
+        # Instantiate a SummaryWriter to output summaries and the Graph.
+        summary_writer = tf.summary.FileWriter("results_tfrecord_nn")
+        summary_writer.add_graph(sess.graph)
+        # Initialize all Variable objects
+        sess.run(init)
+        # actual learning
+        started = t.time()
+        feed_dict = {
+            keep_prob_val: 1.0,
+            keep_prob_train: 0.5,
+            keep_prob_test: 1.0
+        }
+        for i in range(num_epochs):
+            # run training and get the loss
+            _, loss = sess.run([train_optimizer, cross_entropy], feed_dict=feed_dict)
+
+            if i % 100 == 0:
+                print('step {}, loss {} (with dropout)'.format(i, loss))
+                r_accuracy = sess.run([accuracy_val], feed_dict=feed_dict)
+                print("accuracy: {} on validation set (without dropout).".format(r_accuracy))
+                summary_str = sess.run(merged_summary, feed_dict=feed_dict)
+                summary_writer.add_summary(summary_str, i)
+
+        stoped = t.time()
+        accuracy_eval, preds_eval, exp_eval = sess.run([accuracy_test, predictions_test, y_test], feed_dict=feed_dict)
+        print('test accuracy %g' % accuracy_eval)
+        np.set_printoptions(threshold=np.nan)
+        print("Prediction sample: " + str(preds_eval[:50]))
+        print("Actual values: " + str(np.argmax(exp_eval[:50], axis=1)))
+        print("Elapsed time: %.4f s" % (stoped - started))
+
+
+def create_records():
+    import os
+    directory = "/tmp/data/mnist"
+
+    if not os.path.exists(os.path.join(directory, "mnist_train.tfrecords")):
+        convert_to(train, "mnist_train", directory)
+        convert_to(test, "mnist_test", directory)
+        convert_to(val, "mnist_val", directory)
+
+
+if __name__ == '__main__':
+    create_records()
+    train()
+
+
+

skluc/examples/write_read_tfrecords.py

+"""
+Example (with mnist) on how to read and write tf records.
+"""
+
+import tensorflow as tf
+import os
+import numpy as np
+import skluc.mldatasets as dataset
+from skluc.convert_image_to_records import convert_to
+import matplotlib.pyplot as plt
+
+tf.logging.set_verbosity(tf.logging.ERROR)
+
+from collections import namedtuple
+
+dataset_mnist = dataset.MnistDataset()
+dataset_mnist.load()
+mnist = dataset_mnist.to_image()
+
+X_train, Y_train = mnist["train"]
+X_train = np.array(X_train / 255)
+X_test, Y_test = mnist["test"]
+X_test = np.array(X_test / 255)
+
+X_train = X_train.astype(np.float32)
+permut = np.random.permutation(X_train.shape[0])
+val_size = 5000
+X_val = X_train[permut[:val_size]]
+X_train = X_train[permut[val_size:]]
+Y_val = Y_train[permut[:val_size]]
+Y_train = Y_train[permut[val_size:]]
+X_test = X_test.astype(np.float32)
+Y_train = Y_train.astype(np.float32)
+Y_test = Y_test.astype(np.float32)
+
+train = namedtuple("Dataset", ["images", "labels", "num_examples"])
+train.images = X_train
+train.labels = Y_train
+train.num_examples = X_train.shape[0]
+
+test = namedtuple("Dataset", ["images", "labels", "num_examples"])
+test.images = X_test
+test.labels = Y_test
+test.num_examples = X_test.shape[0]
+
+val = namedtuple("Dataset", ["images", "labels", "num_examples"])
+val.images = X_val
+val.labels = Y_val
+val.num_examples = X_val.shape[0]
+
+DIRECTORY = "/tmp/data/mnist"
+
+
+def write_tf_record_mnist():
+    convert_to(train, "mnist_train", DIRECTORY)
+    convert_to(test, "mnist_test", DIRECTORY)
+    convert_to(val, "mnist_val", DIRECTORY)
+
+
+def read_tf_record_mnist():
+    reconstructed_images = []
+    record_iterator = tf.python_io.tf_record_iterator(path=os.path.join(DIRECTORY, "mnist_train.tfrecords"))
+    for string_record in record_iterator:
+        example = tf.train.Example()
+        example.ParseFromString(string_record)
+        height = int(example.features.feature["height"].int64_list.value[0])
+        width = int(example.features.feature["width"].int64_list.value[0])
+        depth = int(example.features.feature["depth"].int64_list.value[0])
+        img_string = (example.features.feature["image_raw"].bytes_list.value[0])
+        label = (example.features.feature["label"].int64_list.value[0])
+
+        img_1d = np.fromstring(img_string, dtype=np.uint32)
+        reconstructed_img = img_1d.reshape((height, width, depth))
+        reconstructed_images.append((reconstructed_img, label))
+
+    plt.imshow(reconstructed_images[0][0][:,:,0])
+    plt.show()
+    print(reconstructed_images[0][1])
+
+
+if __name__ == '__main__':
+    # write_tf_record_mnist()
+    read_tf_record_mnist()
\ No newline at end of file