diff --git a/Code/MonoMultiViewClassifiers/ResultAnalysis.py b/Code/MonoMultiViewClassifiers/ResultAnalysis.py
index 13a165f40e04a5b688c5b878ad7e4192bfab61f8..c8a451db5edb05770d4053e0710e8d77981d3661 100644
--- a/Code/MonoMultiViewClassifiers/ResultAnalysis.py
+++ b/Code/MonoMultiViewClassifiers/ResultAnalysis.py
@@ -90,44 +90,48 @@ def getExampleErrorsBiclass(usedBenchmarkArgumentDictionary, monoviewResults, mu
return exampleErrors
-def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames,minSize=10):
+def plotMetricOneIter(trainScores, testScores, names, nbResults, metricName, fileName, minSize=10):
+ testScores = np.array(testScores)
+ trainScores = np.array(trainScores)
+ names = np.array(names)
+ size = nbResults
+ if nbResults < minSize:
+ size = minSize
+ figKW = {"figsize" : (size, size/3)}
+ f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+ barWidth= 0.35
+ sorted_indices = np.argsort(testScores)
+ testScores = testScores[sorted_indices]
+ trainScores = trainScores[sorted_indices]
+ names = names[sorted_indices]
+
+ ax.set_title(metricName + "\n scores for each classifier")
+ rects = ax.bar(range(nbResults), testScores, barWidth, color="r", )
+ rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", )
+ autolabel(rects, ax, set=1)
+ autolabel(rect2, ax, set=2)
+ ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
+ ax.set_ylim(-0.1, 1.1)
+ ax.set_xticks(np.arange(nbResults) + barWidth)
+ ax.set_xticklabels(names, rotation="vertical")
+ plt.tight_layout()
+ f.savefig(fileName)
+ plt.close()
+
+
+def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames):
for metricName, metricScores in metricsScores.items():
logging.debug("Start:\t Biclass score graph generation for "+metricName)
trainScores = metricScores["trainScores"]
testScores = metricScores["testScores"]
names = metricScores["classifiersNames"]
nbResults = len(testScores)
-
- testScores = np.array(testScores)
- trainScores = np.array(trainScores)
- names = np.array(names)
- size = nbResults
- if nbResults < minSize:
- size = minSize
- figKW = {"figsize" : (size, size/3)}
- f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- barWidth= 0.35
- sorted_indices = np.argsort(testScores)
- testScores = testScores[sorted_indices]
- trainScores = trainScores[sorted_indices]
- names = names[sorted_indices]
-
- ax.set_title(metricName + "\n scores for each classifier")
- rects = ax.bar(range(nbResults), testScores, barWidth, color="r", )
- rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", )
- autolabel(rects, ax, set=1)
- autolabel(rect2, ax, set=2)
- ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
- ax.set_ylim(-0.1, 1.1)
- ax.set_xticks(np.arange(nbResults) + barWidth)
- ax.set_xticklabels(names, rotation="vertical")
- plt.tight_layout()
- f.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-"+"vs".join(labelsNames)+ "-" + metricName + ".png")
- plt.close()
+ fileName = directory + databaseName +"-"+"vs".join(labelsNames)+ "-" + metricName + ".png"
+ plotMetricOneIter(trainScores, testScores, names, nbResults, metricName, fileName)
logging.debug("Done:\t Biclass score graph generation for " + metricName)
-def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames,minSize=10):
+def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames, minSize=10):
logging.debug("Start:\t Biclass Label analysis figure generation")
nbClassifiers = len(exampleErrors)
nbExamples = len(list(exampleErrors.values())[0])
@@ -139,6 +143,7 @@ def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames,min
for iterIndex in range(nbIter):
data[:, classifierIndex * nbIter + iterIndex] = errorOnExamples
temp_data[:,classifierIndex] = errorOnExamples
+
figWidth = max(nbClassifiers/2, minSize)
figHeight = max(nbExamples/20, minSize)
figKW = {"figsize":(figWidth, figHeight)}
@@ -147,7 +152,7 @@ def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames,min
bounds = [-100.5,-0.5, 0.5, 1.5]
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
- cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
+ plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
plt.title('Errors depending on the classifier')
ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
labels = classifiersNames
@@ -155,22 +160,27 @@ def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames,min
red_patch = mpatches.Patch(color='red', label='Classifier failed')
green_patch = mpatches.Patch(color='green', label='Classifier succeded')
black_patch = mpatches.Patch(color='black', label='Unseen data')
- plt.legend(handles=[red_patch, green_patch, black_patch], bbox_to_anchor=(0,1.02,1,0.2), loc="lower left",mode="expand", borderaxespad=0, ncol=3)
+ plt.legend(handles=[red_patch, green_patch, black_patch],
+ bbox_to_anchor=(0,1.02,1,0.2),
+ loc="lower left",
+ mode="expand",
+ borderaxespad=0,
+ ncol=3)
fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-"+"vs".join(labelsNames)+ "-error_analysis.png", bbox_inches="tight")
+ fig.savefig(directory + databaseName +"-"+"vs".join(labelsNames)+ "-error_analysis.png", bbox_inches="tight")
plt.close()
logging.debug("Done:\t Biclass Label analysis figure generation")
logging.debug("Start:\t Biclass Error by example figure generation")
errorOnExamples = -1*np.sum(data, axis=1)/nbIter+nbClassifiers
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors_doubled.csv", data, delimiter=",")
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", temp_data, delimiter=",")
+ np.savetxt(directory + "-clf_errors_doubled.csv", data, delimiter=",")
+ np.savetxt(directory + "-example_errors.csv", temp_data, delimiter=",")
fig, ax = plt.subplots()
x = np.arange(nbExamples)
plt.bar(x, errorOnExamples)
plt.ylim([0,nbClassifiers])
plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-"+"vs".join(labelsNames)+ "-example_errors.png")
+ fig.savefig(directory + databaseName +"-"+"vs".join(labelsNames)+ "-example_errors.png")
plt.close()
logging.debug("Done:\t Biclass Error by example figure generation")
@@ -660,199 +670,200 @@ def getResults(results, statsIter, nbMulticlass, benchmarkArgumentDictionaries,
-def genFusionName(type_, a, b, c):
- """Used to generate fusion classifiers names"""
- if type_ == "Fusion" and a["fusionType"] != "EarlyFusion":
- return "Late-" + str(a["fusionMethod"])
- elif type_ == "Fusion" and a["fusionType"] != "LateFusion":
- return "Early-" + a["fusionMethod"] + "-" + a["classifiersNames"]
-
-def genNamesFromRes(mono, multi):
- """Used to generate classifiers names list (inthe right order) from mono- and multi-view preds"""
- names = [res[1][0] + "-" + res[1][1][-1] for res in mono]
- names += [type_ if type_ != "Fusion" else genFusionName(type_, a, b, c) for type_, a, b, c in multi]
- return names
-
-
-def resultAnalysis(benchmark, results, name, times, metrics, directory, minSize=10):
- """Used to generate bar graphs of all the classifiers scores for each metric """
- mono, multi = results
- for metric in metrics:
- logging.debug("Start:\t Score graph generation for "+metric[0])
- names = genNamesFromRes(mono, multi)
- nbResults = len(mono) + len(multi)
- validationScores = [float(res[1][2][metric[0]][1]) for res in mono]
- validationScores += [float(scores[metric[0]][1]) for a, b, scores, c in multi]
- trainScores = [float(res[1][2][metric[0]][0]) for res in mono]
- trainScores += [float(scores[metric[0]][0]) for a, b, scores, c in multi]
-
- validationScores = np.array(validationScores)
- trainScores = np.array(trainScores)
- names = np.array(names)
- sorted_indices = np.argsort(validationScores)
- validationScores = validationScores[sorted_indices]
- trainScores = trainScores[sorted_indices]
- names = names[sorted_indices]
-
- size = nbResults
- if nbResults < minSize:
- size = minSize
- figKW = {"figsize" : (size, 3.0/4*size+2.0)}
- f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- barWidth= 0.35
- ax.set_title(metric[0] + "\n on validation set for each classifier")
- rects = ax.bar(range(nbResults), validationScores, barWidth, color="r", )
- rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", )
- autolabel(rects, ax)
- autolabel(rect2, ax)
- ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
- ax.set_ylim(-0.1, 1.1)
- ax.set_xticks(np.arange(nbResults) + barWidth)
- ax.set_xticklabels(names, rotation="vertical")
- plt.tight_layout()
- f.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-" + metric[0] + ".png")
- plt.close()
- logging.debug("Done:\t Score graph generation for " + metric[0])
-
-
-def analyzeLabels(labelsArrays, realLabels, results, directory, minSize = 10):
- """Used to generate a graph showing errors on each example depending on classifier"""
- logging.debug("Start:\t Label analysis figure generation")
- mono, multi = results
- classifiersNames = genNamesFromRes(mono, multi)
- nbClassifiers = len(classifiersNames)
- nbExamples = realLabels.shape[0]
- nbIter = 2
- data = np.zeros((nbExamples, nbClassifiers * nbIter))
- tempData = np.array([labelsArray == realLabels for labelsArray in np.transpose(labelsArrays)]).astype(int)
- for classifierIndex in range(nbClassifiers):
- for iterIndex in range(nbIter):
- data[:, classifierIndex * nbIter + iterIndex] = tempData[classifierIndex, :]
- figWidth = max(nbClassifiers/2, minSize)
- figHeight = max(nbExamples/20, minSize)
- figKW = {"figsize":(figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- cmap = mpl.colors.ListedColormap(['red', 'green'])
- bounds = [-0.5, 0.5, 1.5]
- norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
-
- cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
- labels = classifiersNames
- plt.xticks(ticks, labels, rotation="vertical")
- cbar = fig.colorbar(cax, ticks=[0, 1])
- cbar.ax.set_yticklabels(['Wrong', ' Right'])
- fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
- plt.close()
- logging.debug("Done:\t Label analysis figure generation")
-
- logging.debug("Start:\t Error by example figure generation")
- errorOnExamples = -1*np.sum(data, axis=1)/nbIter+nbClassifiers
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors.csv", data, delimiter=",")
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.png")
- plt.close()
- logging.debug("Done:\t Error by example figure generation")
- return data
-
-
-def analyzeIterLabels(labelsAnalysisList, directory, classifiersNames, minSize=10):
- """Used to generate a graph showing errors on each example depending on classifierusing a score
- if multiple iterations"""
- logging.debug("Start:\t Global label analysis figure generation")
- nbExamples = labelsAnalysisList[0].shape[0]
- nbClassifiers = len(classifiersNames)
- nbIter = 2
-
- figWidth = max(nbClassifiers / 2, minSize)
- figHeight = max(nbExamples / 20, minSize)
- figKW = {"figsize": (figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- data = sum(labelsAnalysisList)
- cax = plt.imshow(-data, interpolation='none', cmap="Greys", aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
- plt.xticks(ticks, classifiersNames, rotation="vertical")
- cbar = fig.colorbar(cax, ticks=[0, -len(labelsAnalysisList)])
- cbar.ax.set_yticklabels(['Always Wrong', 'Always Right'])
- fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
- plt.close()
- logging.debug("Done:\t Global label analysis figure generation")
- logging.debug("Start:\t Global error by example figure generation")
- errorOnExamples = -1 * np.sum(data, axis=1) / nbIter + (nbClassifiers*len(labelsAnalysisList))
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors.csv", data, delimiter=",")
- np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers*len(labelsAnalysisList)])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.png")
- plt.close()
- logging.debug("Done:\t Global error by example figure generation")
-
-
-def genFig(iterResults, metric, nbResults, names, nbMono, minSize=10):
- """Used to generate the bar graph representing the mean scores of each classifiers if multiple iteration
- with different random states"""
- nbIter = len(iterResults)
- validationScores = np.zeros((nbIter, nbResults))
- trainScores = np.zeros((nbIter, nbResults))
- for iterIndex, iterResult in enumerate(iterResults):
- mono, multi = iterResult
- validationScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][1]) for res in mono])
- validationScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][1]) for a, b, scores, c in multi])
- trainScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][0]) for res in mono])
- trainScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][0]) for a, b, scores, c in multi])
-
- validationSTDs = np.std(validationScores, axis=0)
- trainSTDs = np.std(trainScores, axis=0)
- validationMeans = np.mean(validationScores, axis=0)
- trainMeans = np.mean(trainScores, axis=0)
- size=nbResults
- if nbResults<minSize:
- size=minSize
- figKW = {"figsize" : (size, 3.0/4*size+2.0)}
- f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- barWidth = 0.35 # the width of the bars
- sorted_indices = np.argsort(validationMeans)
- validationMeans = validationMeans[sorted_indices]
- validationSTDs = validationSTDs[sorted_indices]
- trainSTDs = trainSTDs[sorted_indices]
- trainMeans = trainMeans[sorted_indices]
- names = np.array(names)[sorted_indices]
-
- ax.set_title(metric[0] + " for each classifier")
- rects = ax.bar(range(nbResults), validationMeans, barWidth, color="r", yerr=validationSTDs)
- rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs)
- autolabel(rects, ax)
- autolabel(rect2, ax)
- ax.set_ylim(-0.1, 1.1)
- ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
- ax.set_xticks(np.arange(nbResults) + barWidth)
- ax.set_xticklabels(names, rotation="vertical")
- f.tight_layout()
-
- return f
-
-
-def analyzeIterResults(iterResults, name, metrics, directory):
- nbResults = len(iterResults[0][0]) + len(iterResults[0][1])
- nbMono = len(iterResults[0][0])
- nbIter = len(iterResults)
- names = genNamesFromRes(iterResults[0][0], iterResults[0][1])
- for metric in metrics:
- logging.debug("Start:\t Global score graph generation for " + metric[0])
- figure = genFig(iterResults, metric, nbResults, names, nbMono)
- figure.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-Mean_on_"
- + str(nbIter) + "_iter-" + metric[0] + ".png")
- logging.debug("Done:\t Global score graph generation for " + metric[0])
+# def genFusionName(type_, a, b, c):
+# """Used to generate fusion classifiers names"""
+# if type_ == "Fusion" and a["fusionType"] != "EarlyFusion":
+# return "Late-" + str(a["fusionMethod"])
+# elif type_ == "Fusion" and a["fusionType"] != "LateFusion":
+# return "Early-" + a["fusionMethod"] + "-" + a["classifiersNames"]
+#
+#
+# def genNamesFromRes(mono, multi):
+# """Used to generate classifiers names list (inthe right order) from mono- and multi-view preds"""
+# names = [res[1][0] + "-" + res[1][1][-1] for res in mono]
+# names += [type_ if type_ != "Fusion" else genFusionName(type_, a, b, c) for type_, a, b, c in multi]
+# return names
+#
+#
+# def resultAnalysis(benchmark, results, name, times, metrics, directory, minSize=10):
+# """Used to generate bar graphs of all the classifiers scores for each metric """
+# mono, multi = results
+# for metric in metrics:
+# logging.debug("Start:\t Score graph generation for "+metric[0])
+# names = genNamesFromRes(mono, multi)
+# nbResults = len(mono) + len(multi)
+# validationScores = [float(res[1][2][metric[0]][1]) for res in mono]
+# validationScores += [float(scores[metric[0]][1]) for a, b, scores, c in multi]
+# trainScores = [float(res[1][2][metric[0]][0]) for res in mono]
+# trainScores += [float(scores[metric[0]][0]) for a, b, scores, c in multi]
+#
+# validationScores = np.array(validationScores)
+# trainScores = np.array(trainScores)
+# names = np.array(names)
+# sorted_indices = np.argsort(validationScores)
+# validationScores = validationScores[sorted_indices]
+# trainScores = trainScores[sorted_indices]
+# names = names[sorted_indices]
+#
+# size = nbResults
+# if nbResults < minSize:
+# size = minSize
+# figKW = {"figsize" : (size, 3.0/4*size+2.0)}
+# f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+# barWidth= 0.35
+# ax.set_title(metric[0] + "\n on validation set for each classifier")
+# rects = ax.bar(range(nbResults), validationScores, barWidth, color="r", )
+# rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", )
+# autolabel(rects, ax)
+# autolabel(rect2, ax)
+# ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
+# ax.set_ylim(-0.1, 1.1)
+# ax.set_xticks(np.arange(nbResults) + barWidth)
+# ax.set_xticklabels(names, rotation="vertical")
+# plt.tight_layout()
+# f.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-" + metric[0] + ".png")
+# plt.close()
+# logging.debug("Done:\t Score graph generation for " + metric[0])
+#
+#
+# def analyzeLabels(labelsArrays, realLabels, results, directory, minSize = 10):
+# """Used to generate a graph showing errors on each example depending on classifier"""
+# logging.debug("Start:\t Label analysis figure generation")
+# mono, multi = results
+# classifiersNames = genNamesFromRes(mono, multi)
+# nbClassifiers = len(classifiersNames)
+# nbExamples = realLabels.shape[0]
+# nbIter = 2
+# data = np.zeros((nbExamples, nbClassifiers * nbIter))
+# tempData = np.array([labelsArray == realLabels for labelsArray in np.transpose(labelsArrays)]).astype(int)
+# for classifierIndex in range(nbClassifiers):
+# for iterIndex in range(nbIter):
+# data[:, classifierIndex * nbIter + iterIndex] = tempData[classifierIndex, :]
+# figWidth = max(nbClassifiers/2, minSize)
+# figHeight = max(nbExamples/20, minSize)
+# figKW = {"figsize":(figWidth, figHeight)}
+# fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+# cmap = mpl.colors.ListedColormap(['red', 'green'])
+# bounds = [-0.5, 0.5, 1.5]
+# norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
+#
+# cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
+# plt.title('Errors depending on the classifier')
+# ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
+# labels = classifiersNames
+# plt.xticks(ticks, labels, rotation="vertical")
+# cbar = fig.colorbar(cax, ticks=[0, 1])
+# cbar.ax.set_yticklabels(['Wrong', ' Right'])
+# fig.tight_layout()
+# fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
+# plt.close()
+# logging.debug("Done:\t Label analysis figure generation")
+#
+# logging.debug("Start:\t Error by example figure generation")
+# errorOnExamples = -1*np.sum(data, axis=1)/nbIter+nbClassifiers
+# np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors.csv", data, delimiter=",")
+# np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
+# fig, ax = plt.subplots()
+# x = np.arange(nbExamples)
+# plt.bar(x, errorOnExamples)
+# plt.ylim([0,nbClassifiers])
+# plt.title("Number of classifiers that failed to classify each example")
+# fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.png")
+# plt.close()
+# logging.debug("Done:\t Error by example figure generation")
+# return data
+#
+#
+# def analyzeIterLabels(labelsAnalysisList, directory, classifiersNames, minSize=10):
+# """Used to generate a graph showing errors on each example depending on classifierusing a score
+# if multiple iterations"""
+# logging.debug("Start:\t Global label analysis figure generation")
+# nbExamples = labelsAnalysisList[0].shape[0]
+# nbClassifiers = len(classifiersNames)
+# nbIter = 2
+#
+# figWidth = max(nbClassifiers / 2, minSize)
+# figHeight = max(nbExamples / 20, minSize)
+# figKW = {"figsize": (figWidth, figHeight)}
+# fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+# data = sum(labelsAnalysisList)
+# cax = plt.imshow(-data, interpolation='none', cmap="Greys", aspect='auto')
+# plt.title('Errors depending on the classifier')
+# ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
+# plt.xticks(ticks, classifiersNames, rotation="vertical")
+# cbar = fig.colorbar(cax, ticks=[0, -len(labelsAnalysisList)])
+# cbar.ax.set_yticklabels(['Always Wrong', 'Always Right'])
+# fig.tight_layout()
+# fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
+# plt.close()
+# logging.debug("Done:\t Global label analysis figure generation")
+# logging.debug("Start:\t Global error by example figure generation")
+# errorOnExamples = -1 * np.sum(data, axis=1) / nbIter + (nbClassifiers*len(labelsAnalysisList))
+# np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors.csv", data, delimiter=",")
+# np.savetxt(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
+# fig, ax = plt.subplots()
+# x = np.arange(nbExamples)
+# plt.bar(x, errorOnExamples)
+# plt.ylim([0,nbClassifiers*len(labelsAnalysisList)])
+# plt.title("Number of classifiers that failed to classify each example")
+# fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.png")
+# plt.close()
+# logging.debug("Done:\t Global error by example figure generation")
+#
+#
+# def genFig(iterResults, metric, nbResults, names, nbMono, minSize=10):
+# """Used to generate the bar graph representing the mean scores of each classifiers if multiple iteration
+# with different random states"""
+# nbIter = len(iterResults)
+# validationScores = np.zeros((nbIter, nbResults))
+# trainScores = np.zeros((nbIter, nbResults))
+# for iterIndex, iterResult in enumerate(iterResults):
+# mono, multi = iterResult
+# validationScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][1]) for res in mono])
+# validationScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][1]) for a, b, scores, c in multi])
+# trainScores[iterIndex, :nbMono] = np.array([float(res[1][2][metric[0]][0]) for res in mono])
+# trainScores[iterIndex, nbMono:] = np.array([float(scores[metric[0]][0]) for a, b, scores, c in multi])
+#
+# validationSTDs = np.std(validationScores, axis=0)
+# trainSTDs = np.std(trainScores, axis=0)
+# validationMeans = np.mean(validationScores, axis=0)
+# trainMeans = np.mean(trainScores, axis=0)
+# size=nbResults
+# if nbResults<minSize:
+# size=minSize
+# figKW = {"figsize" : (size, 3.0/4*size+2.0)}
+# f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+# barWidth = 0.35 # the width of the bars
+# sorted_indices = np.argsort(validationMeans)
+# validationMeans = validationMeans[sorted_indices]
+# validationSTDs = validationSTDs[sorted_indices]
+# trainSTDs = trainSTDs[sorted_indices]
+# trainMeans = trainMeans[sorted_indices]
+# names = np.array(names)[sorted_indices]
+#
+# ax.set_title(metric[0] + " for each classifier")
+# rects = ax.bar(range(nbResults), validationMeans, barWidth, color="r", yerr=validationSTDs)
+# rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs)
+# autolabel(rects, ax)
+# autolabel(rect2, ax)
+# ax.set_ylim(-0.1, 1.1)
+# ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
+# ax.set_xticks(np.arange(nbResults) + barWidth)
+# ax.set_xticklabels(names, rotation="vertical")
+# f.tight_layout()
+#
+# return f
+#
+#
+# def analyzeIterResults(iterResults, name, metrics, directory):
+# nbResults = len(iterResults[0][0]) + len(iterResults[0][1])
+# nbMono = len(iterResults[0][0])
+# nbIter = len(iterResults)
+# names = genNamesFromRes(iterResults[0][0], iterResults[0][1])
+# for metric in metrics:
+# logging.debug("Start:\t Global score graph generation for " + metric[0])
+# figure = genFig(iterResults, metric, nbResults, names, nbMono)
+# figure.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + name + "-Mean_on_"
+# + str(nbIter) + "_iter-" + metric[0] + ".png")
+# logging.debug("Done:\t Global score graph generation for " + metric[0])