first commit + commit du document de these

src/models/2019-01/keras_end_to_end_deepstrom_all_datasets/deepstrom_classif_end_to_end.py

+"""
+Benchmark VGG: Benchmarking deepstrom versus other architectures of the VGG network.
+
+Usage:
+    benchmark_classification dense [-q] [--cifar100|--cifar10|--mnist|--svhn] [-f name] [-t size] [-a value] [-v size] [-e numepoch] [-s batchsize] [-D reprdim] [-l size] [-V] [-d val] [--learning-rate val]
+    benchmark_classification deepfriedconvnet [-q] [--cifar100|--cifar10|--mnist|--svhn] [-f name] [-t size] [-a value] [-v size] [-e numepoch] [-s batchsize] [-g gammavalue] [-N nbstack] [-l size] [-z] [-V] [-d val] [--learning-rate val]
+    benchmark_classification deepstrom [-q] [--cifar100|--cifar10|--mnist|--svhn] [-f name] [-t size] [-r] [-a value] [-v size] [-e numepoch] [-s batchsize] [-D reprdim] [-m size] (-R|-L|-C|-E|-P|-S|-A|-T|-M) [-g gammavalue] [-c cvalue] [-n] [-l size] [-V] [-d val] [--learning-rate val]
+
+Options:
+    --help -h                               Display help and exit.
+    -q --quiet                              Set logging level to info.
+    -V --tensorboard                        Write tensorboard logs.
+    -a --seed value                         The seed value used for all randomization processed [default: 0]
+    -t --train-size size                    Size of train set.
+    -v --validation-size size               The size of the validation set [default: 10000]
+    -e --num-epoch=numepoch                 The number of epoch.
+    -s --batch-size=batchsize               The number of example in each batch
+    -d --dropout val                        Keep probability of neurons before classif [default: 1.0]
+    --learning-rate val                     Tell the learning rate for the network [default: 1e-4]
+    -D reprdim --out-dim=reprdim            The dimension of the final representation
+    -f --non-linearity name                 Tell the model which non-linearity to use when necessary (possible values: "relu", "tanh") [default: relu]
+
+Dense:
+    -l --second-layer-size size             Says the size of the second non-linear layer [default: 0]
+
+Deepfried convnet:
+    -N nbstack --nb-stack nbstack           The number of fastfood stack for deepfriedconvnet
+    -z --real-fastfood                      Tell fastfood layer to not update its weights
+
+Deepstrom:
+    -r --real-nystrom                       Says if the matrix for deepstrom should be K^(-1/2)
+    -m size --nys-size size                 The number of example in the nystrom subsample.
+    -n --non-linear                         Tell Nystrom to use the non linear activation function on its output.
+
+Datasets:
+    --cifar10                               Use cifar dataset
+    --mnist                                 Use mnist dataset
+    --svhn                                  Use svhn dataset
+    --cifar100                              Use cifar100 dataset
+
+Possible kernels:
+    -R --rbf-kernel                         Says if the rbf kernel should be used for nystrom.
+    -L --linear-kernel                      Says if the linear kernel should be used for nystrom.
+    -C --chi-square-kernel                  Says if the basic additive chi square kernel should be used for nystrom.
+    -E --exp-chi-square-kernel              Says if the exponential chi square kernel should be used for nystrom.
+    -P --chi-square-PD-kernel               Says if the Positive definite version of the basic additive chi square kernel should be used for nystrom.
+    -S --sigmoid-kernel                     Says it the sigmoid kernel should be used for nystrom.
+    -A --laplacian-kernel                   Says if the laplacian kernel should be used for nystrom.
+    -T --stacked-kernel                     Says if the kernels laplacian, chi2 and rbf in a stacked setting should be used for nystrom.
+    -M --sumed-kernel                       Says if the kernels laplacian, chi2 and rbf in a summed setting should be used for nystrom.
+
+Kernel related:
+    -g gammavalue --gamma gammavalue        The value of gamma for rbf, chi or hyperbolic tangent kernel (deepstrom and deepfriedconvnet)
+    -c cvalue --intercept-constant cvalue   The value of the intercept constant for the hyperbolic tangent kernel.
+
+"""
+
+import time as t
+import docopt
+import numpy as np
+import contextlib
+import sys
+import tensorflow as tf
+import keras
+from keras.callbacks import LearningRateScheduler
+from keras.layers import Dense, BatchNormalization, Flatten, Lambda, Input, Lambda, concatenate
+from keras.optimizers import SGD, Adam
+from keras.preprocessing.image import ImageDataGenerator
+import skluc.main.data.mldatasets as dataset
+from skluc.main.data.utils import normalize, get_uniform_class_rand_indices
+from skluc.main.keras_.kernel import map_kernel_name_function
+from skluc.main.keras_.models import build_lenet_model, build_vgg19_model
+from skluc.main.tensorflow_.kernel_approximation.fastfood_layer import FastFoodLayer
+from skluc.main.tensorflow_.kernel import tf_linear_kernel, tf_rbf_kernel, tf_chi_square_CPD, tf_chi_square_CPD_exp
+from skluc.main.tensorflow_.utils import batch_generator
+from skluc.main.utils import logger, memory_usage, ParameterManager, ResultManager, ResultPrinter, create_directory, compute_euristic_sigma, compute_euristic_sigma_chi2, LoggerWriter
+from src import project_dir
+from sklearn.model_selection import train_test_split
+from keras.models import Sequential, Model
+from keras.layers import Dense, Dropout, Activation, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras.initializers import he_normal
+from skluc.main.keras_.kernel import map_kernel_name_function
+
+from src.features.build_features import _get_model_from_file
+
+def main(paraman, resman, printman):
+
+    def init_number_subsample_bases():
+        '''return: nb_sub_base, zero_padding'''
+        remaining = nys_size % batch_size
+        quotient = nys_size // batch_size
+        if nys_size == 0 or batch_size == 0:
+            raise ValueError
+        if remaining == 0:
+            return quotient, 0
+        elif quotient == 0:
+            return 1, batch_size - remaining
+        else:
+            return quotient + 1, batch_size - remaining
+
+    batch_size = paraman["--batch-size"]  # 128
+    epochs = paraman["--num-epoch"]  # 200
+    nys_size = paraman["--nys-size"]
+    nb_subsample_bases, zero_padding_base = init_number_subsample_bases()
+    kernel = paraman["kernel"]
+
+
+    if paraman["dataset"] == "mnist":
+        data_dir = project_dir / "data/external" / "mnist.npz"
+        build_sequential_model = build_lenet_model
+        datagen = ImageDataGenerator(
+            rotation_range=20,
+            width_shift_range=0.2,
+            height_shift_range=0.2,
+            horizontal_flip=False)
+        num_classes = 10
+    elif paraman["dataset"] == "cifar10":
+        data_dir = project_dir / "data/external" / "cifar10.npz"
+        build_sequential_model = build_vgg19_model
+        datagen = ImageDataGenerator(
+            rotation_range=20,
+            width_shift_range=0.2,
+            height_shift_range=0.2,
+            horizontal_flip=True)
+        num_classes = 10
+
+    elif paraman["dataset"] == "cifar100":
+        data_dir = project_dir / "data/external" / "cifar100.npz"
+        build_sequential_model = build_vgg19_model
+        datagen = ImageDataGenerator(
+            rotation_range=20,
+            width_shift_range=0.2,
+            height_shift_range=0.2,
+            horizontal_flip=True)
+        num_classes = 100
+
+    elif paraman["dataset"] == "svhn":
+        data_dir = project_dir / "data/external" / "svhn.npz"
+        build_sequential_model = build_vgg19_model
+        datagen = ImageDataGenerator(
+            rotation_range=20,
+            width_shift_range=0.2,
+            height_shift_range=0.2,
+            horizontal_flip=False)
+        num_classes = 10
+    else:
+        raise ValueError("Unknown dataset")
+
+    loaded_npz = np.load(data_dir)
+    (x_train, y_train), (x_test, y_test) = (loaded_npz["x_train"], loaded_npz["y_train"]), (loaded_npz["x_test"], loaded_npz["y_test"])
+    input_dim = x_train.shape[1:]
+    output_dim = y_train.shape[1]
+
+    x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=10000, random_state=0)
+
+    y_train = keras.utils.to_categorical(y_train, num_classes)
+    y_val = keras.utils.to_categorical(y_val, num_classes)
+    y_test = keras.utils.to_categorical(y_test, num_classes)
+    x_train = x_train.astype('float32')
+    x_val = x_val.astype('float32')
+    x_test = x_test.astype('float32')
+    y_train = y_train.astype('float32')
+    y_val = y_val.astype('float32')
+    y_test = y_test.astype('float32')
+    x_train = normalize(x_train)  # type: np.ndarray
+    x_val = normalize(x_val)  # type: np.ndarray
+    x_test = normalize(x_test)  # type: np.ndarray
+
+    datagen.fit(x_train)
+
+    subsample_indexes = get_uniform_class_rand_indices(y_train, nys_size)
+    nys_subsample = x_train[subsample_indexes]
+    zero_padding_subsample = np.zeros((zero_padding_base, *nys_subsample.shape[1:]))
+    nys_subsample = np.vstack([nys_subsample, zero_padding_subsample])
+    list_subsample_bases = [nys_subsample[i * batch_size:(i + 1) * batch_size] for i in range(nb_subsample_bases)]
+    kernel_dict = {}
+    if kernel == "rbf":
+        if paraman["--gamma"] is None:
+            logger.debug("Gamma arguments is None. Need to compute it.")
+            gamma_value = 1. / compute_euristic_sigma(x_train)
+        else:
+            gamma_value = float(paraman["--gamma"])
+        kernel_dict = {"gamma": gamma_value}
+    if kernel == "chi2_exp_cpd":
+        if paraman["--gamma"] is None:
+            logger.debug("Gamma arguments is None. Need to compute it.")
+            gamma_value = 1. / compute_euristic_sigma_chi2(x_train)
+        else:
+            gamma_value = float(paraman["--gamma"])
+        kernel_dict = {"gamma": gamma_value}
+
+    # # Model definition
+
+    convmodel_func = build_sequential_model(input_dim)
+
+    # convmodel_func.add(Flatten())
+
+    input_x = Input(shape=input_dim, name="x")
+
+    repr_x = convmodel_func(input_x)
+
+    # todo inserer dense ici
+    input_repr_subsample = [Input(batch_shape=(batch_size, *input_dim)) for _ in range(nb_subsample_bases)]
+    if nb_subsample_bases > 1:
+        input_subsample_concat = concatenate(input_repr_subsample, axis=0)
+    else:
+        input_subsample_concat = input_repr_subsample[0]
+
+    slice_layer = Lambda(lambda input: input[:nys_size],
+                         output_shape=lambda shape: (nys_size, *shape[1:]))
+    input_subsample_concat = slice_layer(input_subsample_concat)
+    reprs_subsample = convmodel_func(input_subsample_concat)
+
+    if kernel == "linear":
+        kernel_function = lambda *args, **kwargs: map_kernel_name_function["linear"](*args, **kwargs, normalize=True, **kernel_dict)
+    elif kernel == "rbf":
+        kernel_function = lambda *args, **kwargs: map_kernel_name_function["rbf"](*args, **kwargs, tanh_activation=True, normalize=True, **kernel_dict)
+    elif kernel == "chi2_cpd":
+        kernel_function = lambda *args, **kwargs: map_kernel_name_function["chi2_cpd"](*args, **kwargs, epsilon=1e-8, tanh_activation=True, normalize=True, **kernel_dict)
+    elif kernel == "chi2_exp_cpd":
+        kernel_function = lambda *args, **kwargs: map_kernel_name_function["chi2_exp_cpd"](*args, **kwargs, epsilon=1e-8, tanh_activation=True, normalize=True, **kernel_dict)
+    else:
+        raise NotImplementedError(f"unknown kernel function {kernel}")
+
+    kernel_layer = Lambda(kernel_function, output_shape=lambda shapes: (shapes[0][0], nys_size))
+    kernel_vector = kernel_layer([repr_x, reprs_subsample])
+
+    input_classifier = Dense(nys_size, use_bias=False, activation='linear')(kernel_vector)  # 512 is the output dim of convolutional layers
+    input_classifier = BatchNormalization()(input_classifier)
+    input_classifier = Dropout(0.5)(input_classifier)
+
+    classif = Dense(num_classes, activation="softmax")(input_classifier)
+
+    model = Model([input_x] + input_repr_subsample, [classif])
+    # sgd = SGD(lr=.1, momentum=0.9, nesterov=True)
+    adam = Adam(lr=.1)
+    model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
+
+    def scheduler(epoch):
+        if epoch < 50:
+            return 1e-3
+        if epoch < 100:
+            return 1e-4
+        return 1e-5
+
+    change_lr = LearningRateScheduler(scheduler)
+    cbks = [change_lr]
+    # calcul de l'accuracy
+    def datagen_food(x, y, x_sub=None, p_datagen=ImageDataGenerator()):
+        if x_sub is None:
+            x_sub = []
+        for x_batch, y_batch in p_datagen.flow(x, y, batch_size=batch_size):
+            if x_batch.shape[0] != batch_size:
+                continue
+            yield [x_batch] + x_sub, y_batch
+
+    start_train = t.time()
+    with contextlib.redirect_stdout(sys.stderr):
+        model.fit_generator(datagen_food(x_train, y_train, list_subsample_bases, datagen),
+                        steps_per_epoch=int(x_train.shape[0]/batch_size),
+                        epochs=epochs,
+                        callbacks=cbks,
+                        verbose=2)
+
+    resman["train_time"] = t.time() - start_train
+
+    start_val = t.time()
+    with contextlib.redirect_stdout(sys.stderr):
+        val_loss, val_acc = model.evaluate_generator(datagen_food(x_val, y_val, list_subsample_bases), steps=int(x_val.shape[0]/batch_size),
+                                                     verbose=2)
+    resman["val_eval_time"] = t.time() - start_val
+    resman["val_acc"] = val_acc
+    
+    start_test = t.time()
+    with contextlib.redirect_stdout(sys.stderr):
+        test_loss, test_acc = model.evaluate_generator(datagen_food(x_test, y_test, list_subsample_bases), steps=int(x_test.shape[0]/batch_size),
+                                                       verbose=2)
+    resman["test_eval_time"] = t.time() - start_test
+    resman["test_acc"] = test_acc
+
+    printman.print()
+
+class ParameterManagerMain(ParameterManager):
+
+    def __init__(self, docopt_dict):
+        super().__init__(docopt_dict)
+
+        self["--out-dim"] = int(self["--out-dim"]) if eval(str(self["--out-dim"])) is not None else None
+        self["kernel"] = self.init_kernel()
+        self["network"] = self.init_network()
+        self["activation_function"] = self.init_non_linearity()
+        self["dataset"] = self.init_dataset()
+        self["--nb-stack"] = int(self["--nb-stack"]) if self["--nb-stack"] is not None else None
+        self["--nys-size"] = int(self["--nys-size"]) if self["--nys-size"] is not None else None
+        self["--num-epoch"] = int(self["--num-epoch"])
+        self["--validation-size"] = int(self["--validation-size"])
+        self["--seed"] = int(self["--seed"])
+        self["--batch-size"] = int(self["--batch-size"])
+        self["--train-size"] = int(self["--train-size"]) if self["--train-size"] is not None else None
+        self["deepstrom_activation"] = self.init_deepstrom_activation()
+        self["--dropout"] = float(self["--dropout"]) if self["--dropout"] is not None else None
+        self["--learning-rate"] = float(self["--learning-rate"]) if self["--learning-rate"] is not None else None
+
+    def init_deepstrom_activation(self):
+        if not self["deepstrom"]:
+            return None
+
+        if self["--non-linear"]:
+            return self["--non-linearity"]
+        else:
+            return None
+
+
+class ResultManagerMain(ResultManager):
+    def __init__(self):
+        super().__init__()
+        self["training_time"] = None
+        self["val_eval_time"] = None
+        self["val_acc"] = None
+        self["test_acc"] = None
+        self["test_eval_time"] = None
+
+
+if __name__ == "__main__":
+    paraman_obj = ParameterManagerMain(docopt.docopt(__doc__))
+    resman_obj = ResultManagerMain()
+    printman_obj = ResultPrinter(paraman_obj, resman_obj)
+    sys.stderr = LoggerWriter(logger.debug)
+    try:
+        main(paraman_obj, resman_obj, printman_obj)
+    except Exception as e:
+        printman_obj.print()
+        raise e

src/models/download_model.py

+# -*- coding: utf-8 -*-
+import click
+from pathlib import Path
+from dotenv import find_dotenv, load_dotenv
+from skluc.main.utils import logger, silentremove, download_data, check_file_md5, DownloadableModel
+
+MAP_NAME_MODEL_VGG19 = {
+    "svhn": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1529968150.5454917_vgg19_svhn.h5",
+        checksum="563a9ec2aad37459bd1ed0e329441b05"
+    ),
+    "cifar100": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1530965727.781668_vgg19_cifar100fine.h5",
+        checksum="edf43e263fec05e2c013dd5a2128fc38"
+    ),
+    "cifar10": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1544802301.9379897_vgg19_Cifar10Dataset.h5",
+        checksum="45714ca91dbbcc1904bea1a10cdcfc7a"
+    ),
+    "siamese_omniglot_28x28": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1536244775.6502118_siamese_vgg19_omniglot_28x28_conv.h5",
+        checksum="90aec06e688ec3248ba89544a10c9f1f"
+    ),
+    "omniglot_28x28": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1536764034.66037_vgg19_omniglot_28x28.h5",
+        checksum="ef1272e9c7ce070e8f70889ec58d1c33"
+    )
+}
+
+
+MAP_NAME_MODEL_LENET = {
+    "mnist": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1524640419.938414_lenet_mnist.h5",
+        checksum="527d7235c213278df1d15d3fe685eb5c"),
+    "siamese_omniglot_28x28": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1536239708.891906_siamese_lenet_omniglot_conv.h5",
+        checksum="5092edcb0be7b31b808e221afcede3e6"
+    ),
+    "omniglot_28x28": DownloadableModel(
+        url="https://pageperso.lis-lab.fr/~luc.giffon/models/1536750152.6389275_lenet_omniglot_28x28.h5",
+        checksum="c4f20b6dae0722234e1ec0bee85e3a4d"
+    )
+}
+
+
+
+MAP_NAME_MAP = {
+    "vgg19": MAP_NAME_MODEL_VGG19,
+    "lenet": MAP_NAME_MODEL_LENET
+}
+
+def _download_all_models(output_dirpath, map_name_model):
+    for key, downloadable_model in map_name_model.items():
+        _download_single_model(Path(output_dirpath) / key, key, map_name_model)
+
+
+def _download_single_model(output_dirpath, weights, map_name_model):
+    output_path = project_dir / output_dirpath
+    s_model_path = download_data(map_name_model[weights].url, output_path)
+    try:
+        check_file_md5(s_model_path, map_name_model[weights].checksum)
+    except ValueError:
+        silentremove(s_model_path)
+
+def _download_single_architecture_weight(output_dirpath, architecture, weights):
+    if weights == "all":
+        _download_all_models(output_dirpath, MAP_NAME_MAP[architecture])
+    else:
+        _download_single_model(Path(output_dirpath) / weights, weights, MAP_NAME_MAP[architecture])
+
+def _download_all_architecture_weight(output_dirpath, weights):
+    for key, map_name_model in MAP_NAME_MAP.items():
+        _download_single_architecture_weight(Path(output_dirpath) / key, weights, map_name_model)
+
+
+@click.command()
+@click.argument('architecture', default="all")
+@click.argument('weights', default="all")
+@click.argument('output_dirpath', type=click.Path())
+def main(weights, architecture, output_dirpath):
+    logger.info(f"Downloading architecture {architecture} with weights {weights} to save to {output_dirpath}")
+
+    if architecture == "all":
+        _download_all_architecture_weight(output_dirpath, weights)
+    else:
+        _download_single_architecture_weight(Path(output_dirpath) / architecture, architecture, weights)
+
+if __name__ == '__main__':
+    # not used in this stub but often useful for finding various files
+    project_dir = Path(__file__).resolve().parents[2]
+
+    # find .env automagically by walking up directories until it's found, then
+    # load up the .env entries as environment variables
+    load_dotenv(find_dotenv())
+
+    main()
+