reformat file: PEP:8

.idea/MNIST_Binary_V2.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/data" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.7 (MNIST_Binary_V2)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

DataLoader/dataLoaders.py

@@ -9,7 +9,7 @@ def get_mnist_dataloaders(batch_size_train, batch_size_test):
     create dataloader for MNIST dataset
     :param batch_size_train: int
     :param batch_size_test: int
-    :return:
+    :return: DataLoader for train, valid and test data
     """
     # train loader
     train_data = datasets.MNIST('./data', train=True, download=True,
@@ -39,8 +39,8 @@ def get_mnist_dataloaders(batch_size_train, batch_size_test):
 
 def get_omniglot_dataloaders_classification(batch_size_train, batch_size_test):
     """
-    Omniglot data set for one-shot learning. This dataset contains 1623 different handwritten characters
-    from 50 different alphabets.
+    Omniglot data set for one-shot learning but used like classification dataset.
+    This dataset contains 1623 different handwritten characters from 50 different alphabets.
     test: 13.180 images
     train: 19.280 images
     image_size = (105, 105)
@@ -69,6 +69,19 @@ def get_omniglot_dataloaders_classification(batch_size_train, batch_size_test):
 
 
 def get_omniglot_dataloader_v2(episodes_per_epoch, n_train, k_train, q_train, n_test, k_test, q_test, dataset_class):
+    """
+    Omniglot data set for one-shot learning.
+    This function, build dataset for few shot learning task.
+    :param episodes_per_epoch:
+    :param n_train:
+    :param k_train:
+    :param q_train:
+    :param n_test:
+    :param k_test:
+    :param q_test:
+    :param dataset_class:
+    :return:
+    """
     background = dataset_class('background')
     background_taskloader = DataLoader(
         background,

DataLoader/dataset.py

@@ -11,7 +11,8 @@ from config import DATA_PATH
 
 class OmniglotDataset(Dataset):
     def __init__(self, subset):
-        """Dataset class representing Omniglot dataset
+        """
+        Dataset class representing Omniglot dataset
         # Arguments:
             subset: Whether the dataset represents the background or evaluation set
         """
@@ -29,8 +30,8 @@ class OmniglotDataset(Dataset):
         self.class_name_to_id = {self.unique_characters[i]: i for i in range(self.num_classes())}
         self.df = self.df.assign(class_id=self.df['class_name'].apply(lambda c: self.class_name_to_id[c]))
         # print(type(self.df.index))
-        # self.df = self.df.assign(class_id=pd.DataFrame(data=self.df.index.values).apply(lambda c: self.class_name_to_id[c]))
-        # print(self.df)
+        # self.df = self.df.assign(class_id=pd.DataFrame(data=self.df.index.values).apply(
+        # lambda c: self.class_name_to_id[c])) print(self.df)
 
         # Create dicts
         # self.datasetid_to_filepath = self.df.to_dict()['id']

DataLoader/prepare_omniglot.py

@@ -12,7 +12,6 @@ from skimage import transform
 import zipfile
 import shutil
 import os
-
 from config import DATA_PATH
 
 # Parameters

experiments/MNIST_binary.py

 import torch.optim as optim
 from DataLoader.dataLoaders import get_mnist_dataloaders
-from utils.training import training, test, gpu_config
+from utils.training import test, run
+from utils.models import get_my_model_MNIST
 from torch import load
 import torch
 
-from utils.models import NoBinaryNetMnist, BinaryNet
+from utils.models import NoBinaryNetMnist
 
 """
 Code source: https://github.com/Wizaron/binary-stochastic-neurons
@@ -14,58 +15,26 @@ Code source: https://github.com/Wizaron/binary-stochastic-neurons
 lr = 0.1
 momentum = 0.9
 nb_epoch = 10
+criterion = F.nll_loss
 batch_size_train = 64
 batch_size_test = 1000
-slope_annealing = False
-reinforce = False
-stochastic = False
-binary = False
-plot_result = False
-first_conv_layer = False
-last_conv_layer = False
 omniglot = True
 
-# Model, activation type, estimator type
-if binary:
-    if stochastic:
-        mode = 'Stochastic'
-        names_model = 'MNIST/Stochastic'
-    else:
-        mode = 'Deterministic'
-        names_model = 'MNIST/Deterministic'
-    if reinforce:
-        estimator = 'REINFORCE'
-        names_model += '_REINFORCE'
-    else:
-        estimator = 'ST'
-        names_model += '_ST'
-    if first_conv_layer:
-        names_model += '_first_conv_binary'
-    if last_conv_layer:
-        names_model += '_last_conv_binary'
-    model = BinaryNet(first_conv_layer=first_conv_layer, last_conv_layer=last_conv_layer,
-                      mode=mode, estimator=estimator)
-else:
-    model = NoBinaryNetMnist()
-    names_model = 'MNIST/NonBinaryNet'
-    mode = None
-    estimator = None
 
 # gpu config:
-model, use_gpu = gpu_config(model)
+model, name_model = get_my_model_MNIST(binary=False)
 
 # MNIST Dataset
 train_loader, valid_loader, test_loader, classes = get_mnist_dataloaders(batch_size_train, batch_size_test)
 
-
 # visualize example
 # show_som_examples(train_loader)
 
 # train
 # optimizer
 optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)
-# train_loss, train_acc, val_loss, val_acc = training(use_gpu, model, names_model, nb_epoch, train_loader, test_loader,
-#                                                    optimizer, plot_result, slope_annealing)
-# test
-model.load_state_dict(load('./trained_models/' + names_model + '.pt', map_location=torch.device('cpu')))
-test_loss, test_acc = test(use_gpu, model, test_loader)
+path_model_checkpoint = ''
+path_save_plot = ''
+log_interval = 2
+run(model, path_model_checkpoint, path_save_plot, name_model, train_loader, valid_loader, nb_epoch, lr, momentum,
+    criterion, log_interval)

experiments/omniglot_binary_classif.py

 import sys
+
 sys.path.append('')
-# import os
-# sys.path.append(os.path.dirname(os.path.realpath('')))
 
-import torch
-from torch import nn
-from torch.optim import SGD
-from torch.utils.data import DataLoader
 import torch.nn.functional as F
-from torchvision.transforms import Compose, ToTensor, Normalize
-from torchvision.datasets import MNIST
-import matplotlib.pyplot as plt
-import numpy as np
-from functools import partial
-
-from utils.models import get_my_model_Omniglot, fetch_last_checkpoint_model_filename
+from utils.models import get_my_model_Omniglot
 from DataLoader.dataLoaders import get_omniglot_dataloaders_classification
-from utils.training import run, evaluate
-
+from utils.training import run
 
 batch_size_train = 64
 batch_size_test = 64
 # Dataset
 train_loader, valid_loader, test_loader = get_omniglot_dataloaders_classification(batch_size_train, batch_size_test)
 
-
 # parameters default values
 epochs = 100
 lr = 1e-3
@@ -40,16 +27,20 @@ print(name_model)
 path_model_checkpoint_no_binary = 'trained_models/Omniglot_classif/No_binary_models/'
 path_save_plot_no_binary = 'results/Omniglot_results/plot_acc_loss/Omniglot_classif/'
 
-run(model_no_binary, path_model_checkpoint_no_binary, path_save_plot_no_binary, name_model, train_loader, valid_loader, epochs, lr, momentum, criterion, log_interval)
-
+print('Begin running No Binary model')
+run(model_no_binary, path_model_checkpoint_no_binary, path_save_plot_no_binary, name_model, train_loader, valid_loader,
+    epochs, lr, momentum, criterion, log_interval)
+print('End running No Binary model')
 
 # parameters model to load no Binary model
 binary = True
 model_binary, name_model = get_my_model_Omniglot(binary)
 print(name_model)
 
-
 path_model_checkpoint_binary = 'trained_models/Omniglot_classif/Binary_models/'
 path_save_plot_binary = 'results/Omniglot_results/plot_acc_loss/Omniglot_classif/'
 
-run(model_binary, path_model_checkpoint_binary, path_save_plot_binary, name_model, train_loader, valid_loader, epochs, lr, momentum, criterion, log_interval)
+print('Begin running Binary model')
+run(model_binary, path_model_checkpoint_binary, path_save_plot_binary, name_model, train_loader, valid_loader, epochs,
+    lr, momentum, criterion, log_interval)
+print('End running Binary model')

requirements.txt

-pandas
-tqdm
-scikit-image
+pandas~=1.0.3
+tqdm~=4.44.1
+scikit-image~=0.16.2
 torch
-torchvision
-matplotlib
-numpy
+torchvision~=0.4.2
+matplotlib~=3.1.3
+numpy~=1.18.1
+pillow~=6.2.1
+ignite
\ No newline at end of file

utils/models.py

 import torch.nn as nn
 import torch.nn.functional as F
 import numpy as np
-
+import os
 from utils.activations import DeterministicBinaryActivation, StochasticBinaryActivation
 from utils.functions import Hardsigmoid
 
 
 # loading the saved model
 def fetch_last_checkpoint_model_filename(model_save_path):
-    import os
+    """
+    fetch last checkpoint model
+    :param model_save_path:
+    :return:
+    """
     checkpoint_files = os.listdir(model_save_path)
     checkpoint_files = [f for f in checkpoint_files if '.pth' in f]
     checkpoint_iter = [
@@ -21,7 +25,15 @@ def fetch_last_checkpoint_model_filename(model_save_path):
 # Model, activation type, estimator type
 def get_my_model_MNIST(binary, stochastic=True, reinforce=False, first_conv_layer=True,
                        last_conv_layer=False):
-
+    """
+    build MNIST model
+    :param binary:
+    :param stochastic:
+    :param reinforce:
+    :param first_conv_layer:
+    :param last_conv_layer:
+    :return:
+    """
     if binary:
         if stochastic:
             mode = 'Stochastic'
@@ -51,9 +63,19 @@ def get_my_model_MNIST(binary, stochastic=True, reinforce=False, first_conv_laye
     return model, names_model
 
 
-# Model, activation type, estimator type
-def get_my_model_Omniglot(binary, stochastic=True, reinforce=False, first_conv_layer=True, second_conv_layer=False, third_conv_layer=False, fourth_conv_layer=False):
-        
+def get_my_model_Omniglot(binary, stochastic=True, reinforce=False, first_conv_layer=True, second_conv_layer=False,
+                          third_conv_layer=False, fourth_conv_layer=False):
+    """
+    build Omniglot model
+    :param binary:
+    :param stochastic:
+    :param reinforce:
+    :param first_conv_layer:
+    :param second_conv_layer:
+    :param third_conv_layer:
+    :param fourth_conv_layer:
+    :return:
+    """
     if binary:
         if stochastic:
             mode = 'Stochastic'
@@ -75,7 +97,9 @@ def get_my_model_Omniglot(binary, stochastic=True, reinforce=False, first_conv_l
             names_model += '_third_conv_binary'
         if fourth_conv_layer:
             names_model += '_fourth_conv_binary'
-        model = BinaryNetOmniglotClassification(first_conv_layer=first_conv_layer, second_conv_layer=second_conv_layer, third_conv_layer=third_conv_layer, fourth_conv_layer=fourth_conv_layer, mode=mode, estimator=estimator)
+        model = BinaryNetOmniglotClassification(first_conv_layer=first_conv_layer, second_conv_layer=second_conv_layer,
+                                                third_conv_layer=third_conv_layer, fourth_conv_layer=fourth_conv_layer,
+                                                mode=mode, estimator=estimator)
     else:
         model = NoBinaryNetOmniglotClassification()
         names_model = 'Omniglot_classif_NonBinaryNet'
@@ -84,9 +108,7 @@ def get_my_model_Omniglot(binary, stochastic=True, reinforce=False, first_conv_l
     return model, names_model
 
 
-
 class Net(nn.Module):
-
     def __init__(self):
         super(Net, self).__init__()
 
@@ -95,7 +117,6 @@ class Net(nn.Module):
 
 
 class NoBinaryNetMnist(Net):
-
     def __init__(self):
         super(NoBinaryNetMnist, self).__init__()
 
@@ -109,8 +130,6 @@ class NoBinaryNetMnist(Net):
         self.act_layer2 = Hardsigmoid()
         self.fc = nn.Linear(7 * 7 * 20, 10)
 
-
-
     def forward(self, input):
         x = input
         slope = 1.0
@@ -125,7 +144,6 @@ class NoBinaryNetMnist(Net):
 
 
 class BinaryNetMNIST(Net):
-
     def __init__(self, first_conv_layer, last_conv_layer, mode='Deterministic', estimator='ST'):
         super(BinaryNetMNIST, self).__init__()
 
@@ -190,28 +208,27 @@ class BinaryNetMNIST(Net):
 
 
 class NoBinaryNetOmniglotClassification(Net):
-
     def __init__(self):
         super(NoBinaryNetOmniglotClassification, self).__init__()
 
-        self.layer1 = nn.Conv2d(1, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm1 = nn.BatchNorm2d(64)
+        self.layer1 = nn.Conv2d(1, 128, kernel_size=3, padding=1, stride=2)
+        self.batchNorm1 = nn.BatchNorm2d(128)
         # self.maxPool1 = nn.MaxPool2d(kernel_size=2, stride=2)
         self.act_layer1 = nn.ReLU()
-        self.layer2 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm2 = nn.BatchNorm2d(64)
+        self.layer2 = nn.Conv2d(128, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm2 = nn.BatchNorm2d(256)
         # self.maxPool2 = nn.MaxPool2d(kernel_size=2, stride=2)            
         self.act_layer2 = nn.ReLU()
-        self.layer3 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm3 = nn.BatchNorm2d(64)
+        self.layer3 = nn.Conv2d(256, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm3 = nn.BatchNorm2d(256)
         # self.maxPool3 = nn.MaxPool2d(kernel_size=2, stride=2)  
         self.act_layer3 = nn.ReLU()
-        self.layer4 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm4 = nn.BatchNorm2d(64)
+        self.layer4 = nn.Conv2d(256, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm4 = nn.BatchNorm2d(256)
         # self.maxPool4 = nn.MaxPool2d(kernel_size=2, stride=2)              
         self.act_layer4 = nn.ReLU()
-        self.fc = nn.Linear(7 * 7 * 64, 1623)
-
+        self.fc1 = nn.Linear(7 * 7 * 256, 4096)
+        self.fc2 = nn.Linear(4096, 1623)
 
     def forward(self, input):
         x = input
@@ -227,14 +244,16 @@ class NoBinaryNetOmniglotClassification(Net):
         x_layer3 = self.act_layer3(self.batchNorm3(self.layer3(x_layer2)))
         x_layer4 = self.act_layer4(self.batchNorm4(self.layer4(x_layer3)))
         x_layer4 = x_layer4.view(x_layer4.size(0), -1)
-        x_fc = self.fc(x_layer4)
-        x_out = F.log_softmax(x_fc, dim=1)
+        x_fc1 = self.fc1(x_layer4)
+        x_fc2 = self.fc2(x_fc1)
+        x_out = F.log_softmax(x_fc2, dim=1)
         return x_out
 
 
 class BinaryNetOmniglotClassification(Net):
 
-    def __init__(self, first_conv_layer, second_conv_layer, third_conv_layer, fourth_conv_layer, mode='Deterministic', estimator='ST'):
+    def __init__(self, first_conv_layer, second_conv_layer, third_conv_layer, fourth_conv_layer, mode='Deterministic',
+                 estimator='ST'):
         super(BinaryNetOmniglotClassification, self).__init__()
 
         assert mode in ['Deterministic', 'Stochastic']
@@ -249,8 +268,8 @@ class BinaryNetOmniglotClassification(Net):
         self.third_conv_layer = third_conv_layer
         self.fourth_conv_layer = fourth_conv_layer
 
-        self.layer1 = nn.Conv2d(1, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm1 = nn.BatchNorm2d(64)
+        self.layer1 = nn.Conv2d(1, 128, kernel_size=3, padding=1, stride=2)
+        self.batchNorm1 = nn.BatchNorm2d(128)
         # self.maxPool1 = nn.MaxPool2d(kernel_size=2, stride=2)
         if self.first_conv_layer:
             if self.mode == 'Deterministic':
@@ -259,8 +278,8 @@ class BinaryNetOmniglotClassification(Net):
                 self.act_layer1 = StochasticBinaryActivation(estimator=estimator)
         else:
             self.act_layer1 = nn.ReLU()
-        self.layer2 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm2 = nn.BatchNorm2d(64)
+        self.layer2 = nn.Conv2d(128, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm2 = nn.BatchNorm2d(256)
         # self.maxPool2 = nn.MaxPool2d(kernel_size=2, stride=2)
         if self.second_conv_layer:
             if self.mode == 'Deterministic':
@@ -269,8 +288,8 @@ class BinaryNetOmniglotClassification(Net):
                 self.act_layer2 = StochasticBinaryActivation(estimator=estimator)
         else:
             self.act_layer2 = nn.ReLU()
-        self.layer3 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm3 = nn.BatchNorm2d(64)
+        self.layer3 = nn.Conv2d(256, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm3 = nn.BatchNorm2d(256)
         # self.maxPool3 = nn.MaxPool2d(kernel_size=2, stride=2) 
         if self.third_conv_layer:
             if self.mode == 'Deterministic':
@@ -279,8 +298,8 @@ class BinaryNetOmniglotClassification(Net):
                 self.act_layer3 = StochasticBinaryActivation(estimator=estimator)
         else:
             self.act_layer3 = nn.ReLU()
-        self.layer4 = nn.Conv2d(64, 64, kernel_size=3, padding=1, stride=2)
-        self.batchNorm4 = nn.BatchNorm2d(64)
+        self.layer4 = nn.Conv2d(256, 256, kernel_size=3, padding=1, stride=2)
+        self.batchNorm4 = nn.BatchNorm2d(256)
         # self.maxPool4 = nn.MaxPool2d(kernel_size=2, stride=2)   
         if self.fourth_conv_layer:
             if self.mode == 'Deterministic':
@@ -289,7 +308,8 @@ class BinaryNetOmniglotClassification(Net):
                 self.act_layer4 = StochasticBinaryActivation(estimator=estimator)
         else:
             self.act_layer4 = nn.ReLU()
-        self.fc = nn.Linear(7 * 7 * 64, 1623)
+        self.fc1 = nn.Linear(7 * 7 * 256, 4096)
+        self.fc2 = nn.Linear(4096, 1623)
 
     def forward(self, input):
         x = input
@@ -329,8 +349,9 @@ class BinaryNetOmniglotClassification(Net):
         else:
             x_layer4 = self.act_layer4(self.batchNorm4(self.layer4(x_layer3) * slope))
         x_layer4 = x_layer4.view(x_layer4.size(0), -1)
-        x_fc = self.fc(x_layer4)
-        x_out = F.log_softmax(x_fc, dim=1)
+        x_fc1 = self.fc1(x_layer4)
+        x_fc2 = self.fc2(x_fc1)
+        x_out = F.log_softmax(x_fc2, dim=1)
         return x_out
 
 

utils/training.py

@@ -19,6 +19,122 @@ from ignite.handlers import ModelCheckpoint
 from tqdm import tqdm
 
 
+#############################
+# Method with Igite pytorch #
+#############################
+
+def run(model, path_model_checkpoint, path_save_plot, name_model, train_loader, val_loader, epochs, lr, momentum,
+        criterion, log_interval, plot_results=True):
+    # train_loader, val_loader = get_data_loaders(train_batch_size, val_batch_size)
+    # model = Net()
+    device = "cpu"
+
+    if torch.cuda.is_available():
+        device = "cuda"
+
+    checkpointer = ModelCheckpoint(path_model_checkpoint, name_model, n_saved=2, create_dir=True,
+                                   save_as_state_dict=True, require_empty=False)
+
+    model.to(device)  # Move model before creating optimizer
+    optimizer = SGD(model.parameters(), lr=lr, momentum=momentum)
+    trainer = create_supervised_trainer(model, optimizer, criterion, device=device)
+    evaluator = create_supervised_evaluator(
+        model, metrics={"accuracy": Accuracy(), "nll": Loss(F.nll_loss)}, device=device
+    )
+
+    training_history = {'accuracy': [], 'loss': []}
+    validation_history = {'accuracy': [], 'loss': []}
+    last_epoch = []
+
+    desc = "ITERATION - loss: {:.2f}"
+    pbar = tqdm(initial=0, leave=False, total=len(train_loader), desc=desc.format(0))
+
+    @trainer.on(Events.ITERATION_COMPLETED(every=log_interval))
+    def log_training_loss(engine):
+        pbar.desc = desc.format(engine.state.output)
+        pbar.update(log_interval)
+
+    @trainer.on(Events.EPOCH_COMPLETED)
+    def log_training_results(engine):
+        pbar.refresh()
+        evaluator.run(train_loader)
+        metrics = evaluator.state.metrics
+        avg_accuracy = metrics["accuracy"]
+        avg_nll = metrics["nll"]
+        accuracy = metrics['accuracy'] * 100
+        training_history['accuracy'].append(accuracy)
+        training_history['loss'].append(avg_nll)
+        tqdm.write(
+            "Training Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
+                engine.state.epoch, accuracy, avg_nll
+            )
+        )
+
+    @trainer.on(Events.EPOCH_COMPLETED)
+    def log_validation_results(engine):
+        evaluator.run(val_loader)
+        metrics = evaluator.state.metrics
+        avg_accuracy = metrics["accuracy"]
+        avg_nll = metrics["nll"]
+        accuracy = metrics['accuracy'] * 100
+        validation_history['accuracy'].append(accuracy)
+        validation_history['loss'].append(avg_nll)
+        tqdm.write(
+            "Validation Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
+                engine.state.epoch, accuracy, avg_nll
+            )
+        )
+
+        pbar.n = pbar.last_print_n = 0
+
+    trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpointer, {'MNIST': model})
+    trainer.run(train_loader, max_epochs=epochs)
+
+    if plot_results:
+        plt.plot(training_history['accuracy'], label="Training Accuracy")
+        plt.plot(validation_history['accuracy'], label="Validation Accuracy")
+        plt.xlabel('No. of Epochs')
+        plt.ylabel('Accuracy')
+        plt.legend(frameon=False)
+        plt.savefig(path_save_plot + name_model + '_acc.png')
+        plt.show()
+
+        plt.plot(training_history['loss'], label="Training Loss")
+        plt.plot(validation_history['loss'], label="Validation Loss")
+        plt.xlabel('No. of Epochs')
+        plt.ylabel('Loss')
+        plt.legend(frameon=False)
+        plt.savefig(path_save_plot + name_model + '_loss.png')
+        plt.show()
+
+    pbar.close()
+
+
+def evaluate(model, test_loader):
+    device = "cpu"
+
+    if torch.cuda.is_available():
+        device = "cuda"
+
+    evaluator_test = create_supervised_evaluator(
+        model, metrics={"accuracy": Accuracy(), "nll": Loss(F.nll_loss)}, device=device
+    )
+
+    evaluator_test.run(test_loader)
+    metrics = evaluator_test.state.metrics
+    avg_nll = metrics["nll"]
+    accuracy = metrics['accuracy'] * 100
+    print(
+        "Test Results - Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
+            accuracy, avg_nll
+        )
+    )
+
+
+###########################
+# Old Method with Pytorch #
+###########################
+
 def categorical_accuracy(y, y_pred):
     """Calculates categorical accuracy.
     # Arguments:
@@ -185,7 +301,6 @@ def plot_loss_acc(loss_values_train, acc_values_train, loss_values_valid, acc_va
     plt.legend(['train', 'test'], loc='upper left')
     plt.savefig(path_save_plot + name_model + '_loss.png')
     plt.show()
-
     return
 
 
@@ -204,7 +319,6 @@ def gradient_step(model: Module, optimiser: optimizer, loss_fn: Callable, x: tor
     loss = loss_fn(y_pred, y)
     loss.backward()
     optimiser.step()
-
     return loss, y_pred
 
 
@@ -224,10 +338,13 @@ def batch_metrics(model: Module, y_pred: torch.Tensor, y: torch.Tensor, metrics:
         else:
             # Assume metric is a callable function
             batch_logs = m(y, y_pred)
-
     return batch_logs
 
 
+#########################################
+# Method for Omniglot few shot learning #
+#########################################
+
 def fit(binary_model, slope_annealing, use_gpu, model: Module, optimiser: optimizer, loss_fn: Callable,
         epochs: int, dataloader: DataLoader,
         prepare_batch: Callable, metrics: List[Union[str, Callable]] = None, callbacks: List[Callback] = None,
@@ -311,111 +428,3 @@ def fit(binary_model, slope_annealing, use_gpu, model: Module, optimiser: optimi
         print('Finished.')
 
     callbacks.on_train_end()
-    
-    
-def run(model, path_model_checkpoint, path_save_plot, name_model, train_loader, val_loader, epochs, lr, momentum, criterion, log_interval, plot_results=True):
-    # train_loader, val_loader = get_data_loaders(train_batch_size, val_batch_size)
-    # model = Net()
-    device = "cpu"
-
-    if torch.cuda.is_available():
-        device = "cuda"
-
-    checkpointer = ModelCheckpoint(path_model_checkpoint, name_model, n_saved=2, create_dir=True, save_as_state_dict=True, require_empty=False)
-
-    model.to(device)  # Move model before creating optimizer
-    optimizer = SGD(model.parameters(), lr=lr, momentum=momentum)
-    trainer = create_supervised_trainer(model, optimizer, criterion, device=device)
-    evaluator = create_supervised_evaluator(
-        model, metrics={"accuracy": Accuracy(), "nll": Loss(F.nll_loss)}, device=device
-    )
-
-    training_history = {'accuracy':[],'loss':[]}
-    validation_history = {'accuracy':[],'loss':[]}
-    last_epoch = []
-
-    desc = "ITERATION - loss: {:.2f}"
-    pbar = tqdm(initial=0, leave=False, total=len(train_loader), desc=desc.format(0))
-
-    @trainer.on(Events.ITERATION_COMPLETED(every=log_interval))
-    def log_training_loss(engine):
-        pbar.desc = desc.format(engine.state.output)
-        pbar.update(log_interval)
-
-
-    @trainer.on(Events.EPOCH_COMPLETED)
-    def log_training_results(engine):
-        pbar.refresh()
-        evaluator.run(train_loader)
-        metrics = evaluator.state.metrics
-        avg_accuracy = metrics["accuracy"]
-        avg_nll = metrics["nll"]
-        accuracy = metrics['accuracy']*100
-        training_history['accuracy'].append(accuracy)
-        training_history['loss'].append(avg_nll)
-        tqdm.write(
-            "Training Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
-                engine.state.epoch, accuracy, avg_nll
-            )
-        )
-
-    @trainer.on(Events.EPOCH_COMPLETED)
-    def log_validation_results(engine):
-        evaluator.run(val_loader)
-        metrics = evaluator.state.metrics
-        avg_accuracy = metrics["accuracy"]
-        avg_nll = metrics["nll"]
-        accuracy = metrics['accuracy']*100     
-        validation_history['accuracy'].append(accuracy)
-        validation_history['loss'].append(avg_nll)
-        tqdm.write(
-            "Validation Results - Epoch: {}  Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
-                engine.state.epoch, accuracy, avg_nll
-            )
-        )
-
-        pbar.n = pbar.last_print_n = 0
-
-    trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpointer, {'MNIST': model})
-    trainer.run(train_loader, max_epochs=epochs)
-
-    if plot_results:
-        plt.plot(training_history['accuracy'],label="Training Accuracy")
-        plt.plot(validation_history['accuracy'],label="Validation Accuracy")
-        plt.xlabel('No. of Epochs')
-        plt.ylabel('Accuracy')
-        plt.legend(frameon=False)
-        plt.savefig(path_save_plot + name_model + '_acc.png')
-        plt.show()
-
-        plt.plot(training_history['loss'],label="Training Loss")
-        plt.plot(validation_history['loss'],label="Validation Loss")
-        plt.xlabel('No. of Epochs')
-        plt.ylabel('Loss')
-        plt.legend(frameon=False)
-        plt.savefig(path_save_plot + name_model + '_loss.png')
-        plt.show()
-
-    pbar.close()
-
-
-def evaluate(model, test_loader):
-    device = "cpu"
-
-    if torch.cuda.is_available():
-        device = "cuda"
-        
-    evaluator_test = create_supervised_evaluator(
-            model, metrics={"accuracy": Accuracy(), "nll": Loss(F.nll_loss)}, device=device
-        )
-
-    evaluator_test.run(test_loader)
-    metrics = evaluator_test.state.metrics
-    avg_nll = metrics["nll"]
-    accuracy = metrics['accuracy']*100     
-    print(
-        "Test Results - Avg accuracy: {:.2f} Avg loss: {:.2f}".format(
-            accuracy, avg_nll
-        )
-    )
-

visualize/layer_activation_with_guided_backprop.py

-"""
-Created on Thu Oct 26 11:23:47 2017
-@author: Utku Ozbulak - github.com/utkuozbulak
-"""
-import torch
-from torch.nn import ReLU
-
-
-class GuidedBackprop():
-    """
-       Produces gradients generated with guided back propagation from the given image
-    """
-    def __init__(self, model):
-        self.model = model
-        self.gradients = None
-        self.forward_relu_outputs = []
-        # Put model in evaluation mode
-        self.model.eval()
-        self.update_relus()
-        self.hook_layers()
-
-    def hook_layers(self):
-        def hook_function(module, grad_in, grad_out):
-            self.gradients = grad_in[0]
-        # Register hook to the first layer
-        first_layer = list(self.model.features._modules.items())[0][1]
-        first_layer.register_backward_hook(hook_function)
-
-    def update_relus(self):
-        """
-            Updates relu activation functions so that
-                1- stores output in forward pass
-                2- imputes zero for gradient values that are less than zero
-        """
-        def relu_backward_hook_function(module, grad_in, grad_out):
-            """
-            If there is a negative gradient, change it to zero
-            """
-            # Get last forward output
-            corresponding_forward_output = self.forward_relu_outputs[-1]
-            corresponding_forward_output[corresponding_forward_output > 0] = 1
-            modified_grad_out = corresponding_forward_output * torch.clamp(grad_in[0], min=0.0)
-            del self.forward_relu_outputs[-1]  # Remove last forward output
-            return (modified_grad_out,)
-
-        def relu_forward_hook_function(module, ten_in, ten_out):
-            """
-            Store results of forward pass
-            """
-            self.forward_relu_outputs.append(ten_out)
-
-        # Loop through layers, hook up ReLUs
-        for pos, module in self.model.features._modules.items():
-            if isinstance(module, ReLU):
-                module.register_backward_hook(relu_backward_hook_function)
-                module.register_forward_hook(relu_forward_hook_function)
-
-    def generate_gradients(self, input_image, target_class, cnn_layer, filter_pos):
-        self.model.zero_grad()
-        # Forward pass
-        x = input_image
-        for index, layer in enumerate(self.model.features):
-            # Forward pass layer by layer
-            # x is not used after this point because it is only needed to trigger
-            # the forward hook function
-            x = layer(x)
-            # Only need to forward until the selected layer is reached
-            if index == cnn_layer:
-                # (forward hook function triggered)
-                break
-        conv_output = torch.sum(torch.abs(x[0, filter_pos]))
-        # Backward pass
-        conv_output.backward()
-        # Convert Pytorch variable to numpy array
-        # [0] to get rid of the first channel (1,3,224,224)
-        gradients_as_arr = self.gradients.data.numpy()[0]
-        return gradients_as_arr

visualize/viz.py

 import matplotlib.pyplot as plt
 import numpy as np
+import torchvision
 from torchvision.utils import make_grid
 from torch import no_grad, max
 import torch
@@ -189,11 +190,11 @@ class GradientAscent:
         self.first_conv_layer = first_conv_layer
 
         if not self.mean_gradient:
-          assert self.ind_x != None and self.ind_y != None, 'if mean_gradient is false, you must choice x and y index'
+            assert self.ind_x is not None and self.ind_y is not None, 'if mean_gradient is false, you must choice x ' \
+                                                                      'and y index '
 
-
-        if (type(filter_idxs) == int):
-            output = self._visualize_filter(layer,
+        if type(filter_idxs) == int:
+            self._visualize_filter(layer,
                                    filter_idxs,
                                    self.mean_gradient,
                                    self.ind_x,
@@ -235,6 +236,7 @@ class GradientAscent:
             else:
                 # maximization of a specific neuron for filter_idx
                 self.activation = output[:, filter_idx, ind_x, ind_y]
+
         return layer.register_forward_hook(_record_activation)
 
     def _register_backward_hooks(self):
@@ -257,8 +259,6 @@ class GradientAscent:
                 torch.mul(self.gradients, self.gradients))) + 1e-5)
             x = x + self.gradients * self._lr
             output.append(x)
-            # TODO: regarder loss et acc pour voir si รงa fonctionne
-
         return output
 
     def _validate_filter_idx(self, num_filters, filter_idx):
@@ -267,7 +267,8 @@ class GradientAscent:
         elif (filter_idx < 0) or (filter_idx > num_filters):
             raise ValueError(f'Filter index must be between 0 and {num_filters - 1}.')
 
-    def _visualize_filter(self, layer, filter_idx, mean_gradient, ind_x, ind_y, first_conv_layer, num_iter, figsize, title):
+    def _visualize_filter(self, layer, filter_idx, mean_gradient, ind_x, ind_y, first_conv_layer, num_iter, figsize,
+                          title):
         self.output = self.optimize(layer, filter_idx, mean_gradient, ind_x, ind_y, num_iter=num_iter)
 
         plt.figure(figsize=figsize)
@@ -289,7 +290,8 @@ class GradientAscent:
         plt.show()
         # plt.imsave('plot_image_maximize_filter_layer2_model_MNIST.png')
 
-    def _visualize_filters(self, layer, filter_idxs, mean_gradient, ind_x, ind_y, first_conv_layer, num_iter, num_subplots,
+    def _visualize_filters(self, layer, filter_idxs, mean_gradient, ind_x, ind_y, first_conv_layer, num_iter,
+                           num_subplots,
                            title):
         # Prepare the main plot
 
@@ -300,7 +302,6 @@ class GradientAscent:
         plt.title(title)
         plt.axis('off')
 
-
         self.output = []
 
         # Plot subplots
@@ -314,7 +315,6 @@ class GradientAscent:
             ax.set_yticks([])
             ax.set_title(f'filter {filter_idx}')
 
-            
             ax.imshow(format_for_plotting(
                 standardize_and_clip(output[-1],
                                      saturation=0.15,
@@ -360,8 +360,8 @@ def viz_activations(model, loader, index_data=None):
       plt.show()
       """
 
-def viz_filters(model):
 
+def viz_filters(model):
     for name, m in model.named_modules():
         if type(m) == nn.Conv2d:
             filters = m.weight.data.clone()
@@ -372,7 +372,6 @@ def viz_filters(model):
 
 
 def viz_heatmap(model, name_model, loader, index_data=None, save=True):
-
     activation = {}
     for name, m in model.named_modules():
         if type(m) == Hardsigmoid or type(m) == nn.ReLU:
@@ -462,7 +461,6 @@ def imshow_v2(inp, title=None):
 
 
 def imshow_v3(img, title):
-  
     """Custom function to display the image using matplotlib"""
 
     # define std correction to be made
@@ -489,42 +487,6 @@ def show_databatch(inputs, classes):
     print(classes)
 
 
-def show_batch_images(dataloader):
-
-  images,_ = next(iter(dataloader))
-  
-  #run the model on the images
-  outputs = model((images, 1.0))
-  
-  #get the maximum class 
-  _, pred = torch.max(outputs.data, 1)
-  
-  #make grid
-  img = torchvision.utils.make_grid(images)
-  
-  #call the function
-  imshow(img, title=[classes[x.item()] for x in pred])
-  
-  return images, pred
-  
-  
-#custom function to fetch images from dataloader
-def show_simple_image(dataloader):
-  images,_ = next(iter(dataloader))
-  
-  #run the model on the images
-  outputs = model((images, 1.0))
-  
-  #get the maximum class 
-  _, pred = torch.max(outputs.data, 1)
-
-  
-  #call the function
-  # imshow(images[0], title=[classes[x.item()] for x in pred])
-  
-  return images[0]
-  
-
 def visualize_model(model, dataloaders, device, num_images=6):
     was_training = model.training
     model.eval()
@@ -552,12 +514,6 @@ def visualize_model(model, dataloaders, device, num_images=6):
         model.train(mode=was_training)
 
 
-def show_som_examples(loader):
-    # visualize some example of dataset
-    inputs, classes = next(iter(loader))
-    show_databatch(inputs, classes)
-
-
 def visTensor(tensor, ch=0, allkernels=False, nrow=8, padding=1):
     n, c, w, h = tensor.shape
 
@@ -653,10 +609,8 @@ def format_for_plotting(tensor):
         return formatted.permute(1, 2, 0).detach()
 
 
-        
 def standardize_and_clip(tensor, min_value=0.0, max_value=1.0,
                          saturation=0.1, brightness=0.5):
-
     """Standardizes and clips input tensor.
     Standardizes the input tensor (mean = 0.0, std = 1.0). The color saturation
     and brightness are adjusted, before tensor values are clipped to min/max
@@ -856,7 +810,6 @@ def get_filter_layer3():
                       [1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1]]))
 
 
-
 def get_region_layer4(image, ind_x, ind_y, name, stride, padding, filter_size, len_img_h, len_img_w):
     """
   return region of interest from index (x,y)
@@ -902,9 +855,9 @@ def get_region_layer4(image, ind_x, ind_y, name, stride, padding, filter_size, l
         reduice_region_raw_size = 0
 
     begin_col = index_col_hl_4
-  end_col = index_col_hl_4 + ((((((filter_size*stride)+1)*stride)+1))*stride)+1 + reduice_region_col_size
+    end_col = index_col_hl_4 + (((((filter_size * stride) + 1) * stride) + 1) * stride) + 1 + reduice_region_col_size
     begin_raw = index_raw_hl_4
-  end_raw = index_raw_hl_4 + ((((((filter_size*stride)+1)*stride)+1))*stride)+1 + reduice_region_raw_size
+    end_raw = index_raw_hl_4 + (((((filter_size * stride) + 1) * stride) + 1) * stride) + 1 + reduice_region_raw_size
 
     if end_col > len_img_w:
         end_col = len_img_w
@@ -952,9 +905,7 @@ def get_filter_layer4():
                       [1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1]]))
 
 
-
 def get_all_regions_max(loader, activations):
-
     dataiter = iter(loader)
     images, _ = dataiter.next()
     print('nb images: {}'.format(len(images)))
@@ -1061,7 +1012,8 @@ def get_regions_interest(regions, activation, best, worst, viz_mean_img, viz_gri
         assert percentage <= 100 and percentage >= 0, 'percentage value must be in 0 and 100'
         n = int((len(activation) * percentage) / 100)
         print('Consider {}% image regions = {} images'.format(percentage, n))
-    selected_regions = get_n_first_regions_index(best, worst, n, activations_values_interest, nb_filter, regions_interest_filter)
+        selected_regions = get_n_first_regions_index(best, worst, n, activations_values_interest, nb_filter,
+                                                     regions_interest_filter)
 
     nb_regions = selected_regions[0].shape[0]
 
@@ -1070,7 +1022,8 @@ def get_regions_interest(regions, activation, best, worst, viz_mean_img, viz_gri
         nb_image = 1
         print('mean image:')
         for i, ind_filter in enumerate(list_filter):
-        print('mean regions of {} regions more={} or worst={} active for filter number: {} :'.format(n, best, worst, ind_filter))
+            print('mean regions of {} regions more={} or worst={} active for filter number: {} :'.format(n, best, worst,
+                                                                                                         ind_filter))
             mean_img = np.mean(selected_regions[i], 0)
             viz_regions(nb_image, mean_img)
 
@@ -1079,7 +1032,8 @@ def get_regions_interest(regions, activation, best, worst, viz_mean_img, viz_gri
         print('grid image')
         for i, ind_filter in enumerate(list_filter):
             region_to_print = []
-        print('grid regions of {} regions more={} or worst={} active for filter number: {} :'.format(n, best, worst, ind_filter))
+            print('grid regions of {} regions more={} or worst={} active for filter number: {} :'.format(n, best, worst,
+                                                                                                         ind_filter))
             for j in range(nb_regions):
                 region_to_print.append(selected_regions[i][j])
             viz_regions(nb_image, region_to_print)