ResultAnalysis.py

# Import built-in modules
import time
import pylab
import logging

# Import third party modules
import matplotlib

matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
import matplotlib as mpl

# Import own Modules
import Metrics
from utils.Transformations import signLabels

# Author-Info
__author__ = "Baptiste Bauvin"
__status__ = "Prototype"  # Production, Development, Prototype


def autolabel(rects, ax):
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width() / 2., 1.01 * height,
                "%.2f" % height,
                ha='center', va='bottom')


def genFusionName(type_, a, b, c):
    if type_ == "Fusion" and a["fusionType"] != "EarlyFusion":
        return "Late-" + str(a["fusionMethod"])
    elif type_ == "Fusion" and a["fusionType"] != "LateFusion":
        return "Early-" + a["fusionMethod"] + "-" + a["classifiersNames"]


def genNamesFromRes(mono, multi):
    names = [res[1][0] + "-" + res[1][1][-1] for res in mono]
    names += [type_ if type_ != "Fusion" else genFusionName(type_, a, b, c) for type_, a, b, c in multi]
    return names


def resultAnalysis(benchmark, results, name, times, metrics, directory, minSize=10):
    mono, multi = results
    for metric in metrics:
        names = genNamesFromRes(mono, multi)
        nbResults = len(mono) + len(multi)
        validationScores = [float(res[1][2][metric[0]][1]) for res in mono]
        validationScores += [float(scores[metric[0]][1]) for a, b, scores, c in multi]
        trainScores = [float(res[1][2][metric[0]][0]) for res in mono]
        trainScores += [float(scores[metric[0]][0]) for a, b, scores, c in multi]

        validationScores = np.array(validationScores)
        trainScores = np.array(trainScores)
        names = np.array(names)
        size = nbResults
        if nbResults < minSize:
            size = minSize
        figKW = {"figsize" : (size, 3.0/4*size+2.0)}
        f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
        barWidth= 0.35
        sorted_indices = np.argsort(validationScores)
        validationScores = validationScores[sorted_indices]
        trainScores = trainScores[sorted_indices]
        names = names[sorted_indices]

        ax.set_title(metric[0] + "\n on validation set for each classifier")
        rects = ax.bar(range(nbResults), validationScores, barWidth, color="r", )
        rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", )
        autolabel(rects, ax)
        autolabel(rect2, ax)
        ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
        ax.set_ylim(-0.1, 1.1)
        ax.set_xticks(np.arange(nbResults) + barWidth)
        ax.set_xticklabels(names, rotation="vertical")
        plt.tight_layout()
        f.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-" + metric[0] + ".png")
        plt.close()


def analyzeLabels(labelsArrays, realLabels, results, directory):
    mono, multi = results
    classifiersNames = genNamesFromRes(mono, multi)
    nbClassifiers = len(classifiersNames)
    nbExamples = realLabels.shape[0]
    nbIter = 2
    data = np.zeros((nbExamples, nbClassifiers * nbIter))
    tempData = np.array([labelsArray == realLabels for labelsArray in np.transpose(labelsArrays)]).astype(int)
    for classifierIndex in range(nbClassifiers):
        for iterIndex in range(nbIter):
            data[:, classifierIndex * nbIter + iterIndex] = tempData[classifierIndex, :]
    figKW = {"figsize":(nbClassifiers/2, nbExamples/20)}
    fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
    cmap = mpl.colors.ListedColormap(['red', 'green'])
    bounds = [-0.5, 0.5, 1.5]
    norm = mpl.colors.BoundaryNorm(bounds, cmap.N)

    cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
    plt.title('Error on examples depending on the classifier')
    ticks = np.arange(0, nbClassifiers * nbIter, nbIter)
    labels = classifiersNames
    plt.xticks(ticks, labels, rotation="vertical")
    cbar = fig.colorbar(cax, ticks=[0, 1])
    cbar.ax.set_yticklabels(['Wrong', ' Right'])
    fig.tight_layout()
    fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
    plt.close()


def genScoresNames(iterResults, metric, nbResults, names, nbMono, minSize=10):
    nbIter = len(iterResults)
    validationScores = np.zeros((nbIter, nbResults))
    trainScores = np.zeros((nbIter, nbResults))
    for iterIndex, iterResult in enumerate(iterResults):
        mono, multi = iterResult
        validationScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][1]) for res in mono])
        validationScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][1]) for a, b, scores, c in multi])
        trainScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][0]) for res in mono])
        trainScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][0]) for a, b, scores, c in multi])

    validationSTDs = np.std(validationScores, axis=0)
    trainSTDs = np.std(trainScores, axis=0)
    validationMeans = np.mean(validationScores, axis=0)
    trainMeans = np.mean(trainScores, axis=0)
    size=nbResults
    if nbResults<minSize:
        size=minSize
    figKW = {"figsize" : (size, 3.0/4*size+2.0)}
    f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
    barWidth = 0.35  # the width of the bars
    sorted_indices = np.argsort(validationMeans)
    validationMeans = validationMeans[sorted_indices]
    validationSTDs = validationSTDs[sorted_indices]
    trainSTDs = trainSTDs[sorted_indices]
    trainMeans = trainMeans[sorted_indices]
    names = np.array(names)[sorted_indices]

    ax.set_title(metric[0] + " for each classifier")
    rects = ax.bar(range(nbResults), validationMeans, barWidth, color="r", yerr=validationSTDs)
    rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs)
    autolabel(rects, ax)
    autolabel(rect2, ax)
    ax.set_ylim(-0.1, 1.1)
    ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
    ax.set_xticks(np.arange(nbResults) + barWidth)
    ax.set_xticklabels(names, rotation="vertical")
    f.tight_layout()

    return f


def analyzeIterResults(iterResults, name, metrics, directory):
    nbResults = len(iterResults[0][0]) + len(iterResults[0][1])
    nbMono = len(iterResults[0][0])
    nbIter = len(iterResults)
    names = genNamesFromRes(iterResults[0][0], iterResults[0][1])
    for metric in metrics:
        figure = genScoresNames(iterResults, metric, nbResults, names, nbMono)
        figure.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-Mean_on_"
                       + str(nbIter) + "_iter-" + metric[0] + ".png")