From 3337494326eb7909bb4e581ab14ea61f4dd92758 Mon Sep 17 00:00:00 2001
From: Baptiste Bauvin <baptiste.bauvin@lis-lab.fr>
Date: Mon, 13 Aug 2018 11:47:39 -0400
Subject: [PATCH] Simplified error analysis plotting
---
.../ResultAnalysis.py | 283 +++++++-----------
1 file changed, 116 insertions(+), 167 deletions(-)
diff --git a/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py b/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py
index 2ad258dd..192a4faf 100644
--- a/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py
+++ b/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py
@@ -119,7 +119,7 @@ def getExampleErrorsBiclass(groud_truth, results):
errorOnExamples = np.equal(classifierResult.full_labels_pred, groud_truth).astype(int)
unseenExamples = np.where(groud_truth==-100)[0]
errorOnExamples[unseenExamples]=-100
- exampleErrors[classifierResult.get_classifier_name()] = errorOnExamples
+ exampleErrors[classifierResult.get_classifier_name()] = {"errorOnExamples": errorOnExamples}
return exampleErrors
@@ -215,6 +215,12 @@ def plotMetricScores(trainScores, testScores, names, nbResults, metricName, file
The plotted metric's name
fileName : str
The name of the file where the figure will be saved.
+ tag : str
+ Some text to personalize the title, must start with a whitespace.
+ train_STDs : np.array of floats or None
+ The array containing the standard deviations for the averaged scores on the training set.
+ test_STDs : np.array of floats or None
+ The array containing the standard deviations for the averaged scores on the testing set.
Returns
-------
@@ -228,8 +234,8 @@ def plotMetricScores(trainScores, testScores, names, nbResults, metricName, file
f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
ax.set_title(metricName + "\n"+ tag +" scores for each classifier")
- rects = ax.bar(range(nbResults), testScores, barWidth, color="0.8", yerr=test_STDs)
- rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.5", yerr=train_STDs)
+ rects = ax.bar(range(nbResults), testScores, barWidth, color="0.1", yerr=test_STDs)
+ rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.8", yerr=train_STDs)
autolabel(rects, ax, set=1, std=test_STDs)
autolabel(rect2, ax, set=2, std=train_STDs)
@@ -274,58 +280,81 @@ def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames):
logging.debug("Done:\t Biclass score graph generation for " + metricName)
-def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames, minSize=10):
- logging.debug("Start:\t Biclass Label analysis figure generation")
- nbClassifiers = len(exampleErrors)
- nbExamples = len(list(exampleErrors.values())[0])
- nbIter = 2
- data = np.zeros((nbExamples, nbClassifiers * nbIter))
- temp_data = np.zeros((nbExamples, nbClassifiers))
- classifiersNames = exampleErrors.keys()
- for classifierIndex, (classifierName, errorOnExamples) in enumerate(exampleErrors.items()):
- for iterIndex in range(nbIter):
- data[:, classifierIndex * nbIter + iterIndex] = errorOnExamples
- temp_data[:,classifierIndex] = errorOnExamples
-
- figWidth = max(nbClassifiers/2, minSize)
- figHeight = max(nbExamples/20, minSize)
- figKW = {"figsize":(figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- cmap = mpl.colors.ListedColormap(['black', 'red', 'green'])
- bounds = [-100.5,-0.5, 0.5, 1.5]
+def iterCmap(statsIter):
+ cmapList = ["red", "0.0"]+[str(float((i+1))/statsIter) for i in range(statsIter)]
+ cmap = mpl.colors.ListedColormap(cmapList)
+ bounds = [-100*statsIter-0.5, -0.5]
+ for i in range(statsIter):
+ bounds.append(i+0.5)
+ bounds.append(statsIter+0.5)
norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
+ return cmap, norm
+
- plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
+def publish2Dplot(data, classifiersNames, nbClassifiers, nbExamples, nbCopies, fileName, minSize=10,
+ width_denominator=2.0, height_denominator=20.0, statsIter=1):
+ figWidth = max(nbClassifiers / width_denominator, minSize)
+ figHeight = max(nbExamples / height_denominator, minSize)
+ figKW = {"figsize": (figWidth, figHeight)}
+ fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+ cmap, norm = iterCmap(statsIter)
+ cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
plt.title('Errors depending on the classifier')
- ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
+ ticks = np.arange(nbCopies / 2 - 0.5, nbClassifiers * nbCopies, nbCopies)
labels = classifiersNames
plt.xticks(ticks, labels, rotation="vertical")
- red_patch = mpatches.Patch(color='red', label='Classifier failed')
- green_patch = mpatches.Patch(color='green', label='Classifier succeded')
- black_patch = mpatches.Patch(color='black', label='Unseen data')
- plt.legend(handles=[red_patch, green_patch, black_patch],
- bbox_to_anchor=(0,1.02,1,0.2),
- loc="lower left",
- mode="expand",
- borderaxespad=0,
- ncol=3)
+ cbar = fig.colorbar(cax, ticks=[-100 * statsIter / 2, 0, statsIter])
+ cbar.ax.set_yticklabels(['Unseen', 'Always Wrong', 'Always Right'])
fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-"+"_vs_".join(labelsNames)+ "-error_analysis.png", bbox_inches="tight")
+ fig.savefig(fileName+"error_analysis_2D.png", bbox_inches="tight")
plt.close()
- logging.debug("Done:\t Biclass Label analysis figure generation")
- logging.debug("Start:\t Biclass Error by example figure generation")
- errorOnExamples = -1*np.sum(data, axis=1)/nbIter+nbClassifiers
- np.savetxt(directory + "clf_errors_doubled.csv", data, delimiter=",")
- np.savetxt(directory + "example_errors.csv", temp_data, delimiter=",")
+
+def publishErrorsBarPlot(errorOnExamples, nbClassifiers, nbExamples, fileName):
fig, ax = plt.subplots()
x = np.arange(nbExamples)
plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers])
+ plt.ylim([0, nbClassifiers])
plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-"+"_vs_".join(labelsNames)+ "-example_errors.png")
+ fig.savefig(fileName+"error_analysis_bar.png")
plt.close()
- logging.debug("Done:\t Biclass Error by example figure generation")
+
+
+def gen_error_data(example_errors, base_file_name, nbCopies=2):
+ nbClassifiers = len(example_errors)
+ nbExamples = len(list(example_errors.values())[0]["errorOnExamples"])
+ classifiersNames = example_errors.keys()
+
+ data = np.zeros((nbExamples, nbClassifiers * nbCopies))
+ temp_data = np.zeros((nbExamples, nbClassifiers))
+ for classifierIndex, (classifierName, errorOnExamples) in enumerate(example_errors.items()):
+ for iterIndex in range(nbCopies):
+ data[:, classifierIndex * nbCopies + iterIndex] = errorOnExamples["errorOnExamples"]
+ temp_data[:, classifierIndex] = errorOnExamples["errorOnExamples"]
+ errorOnExamples = -1 * np.sum(data, axis=1) / nbCopies + nbClassifiers
+
+ np.savetxt(base_file_name + "2D_plot_data.csv", data, delimiter=",")
+ np.savetxt(base_file_name + "bar_plot_data.csv", temp_data, delimiter=",")
+
+ return nbClassifiers, nbExamples, nbCopies, classifiersNames, data, errorOnExamples
+
+
+def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames):
+
+ logging.debug("Start:\t Biclass Label analysis figure generation")
+
+ base_file_name = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName + "-" + "_vs_".join(
+ labelsNames) + "-"
+
+ nbClassifiers, nbExamples, nCopies, classifiersNames, data, errorOnExamples = gen_error_data(exampleErrors,
+ base_file_name)
+
+
+ publish2Dplot(data, classifiersNames, nbClassifiers, nbExamples, nCopies, base_file_name)
+
+ publishErrorsBarPlot(errorOnExamples, nbClassifiers, nbExamples, base_file_name)
+
+ logging.debug("Done:\t Biclass Label analysis figures generation")
def analyzeBiclass(results, benchmarkArgumentDictionaries, statsIter, metrics):
@@ -386,7 +415,7 @@ def getErrorOnLabelsMulticlass(multiclassResults, multiclassLabels):
for iterIndex, iterResults in enumerate(multiclassResults):
for classifierName, classifierResults in iterResults.items():
errorOnExamples = classifierResults["labels"] == multiclassLabels
- multiclassResults[iterIndex][classifierName]["errorOnExample"] = errorOnExamples.astype(int)
+ multiclassResults[iterIndex][classifierName]["errorOnExamples"] = errorOnExamples.astype(int)
logging.debug("Done:\t Getting errors on each example for each classifier")
@@ -416,50 +445,18 @@ def publishMulticlassScores(multiclassResults, metrics, statsIter, direcories, d
def publishMulticlassExmapleErrors(multiclassResults, directories, databaseName, minSize=10):
for iterIndex, multiclassResult in enumerate(multiclassResults):
directory = directories[iterIndex]
- logging.debug("Start:\t Label analysis figure generation")
- nbClassifiers = len(multiclassResult)
- nbExamples = len(list(multiclassResult.values())[0]["errorOnExample"])
- nbIter = 2
- data = np.zeros((nbExamples, nbClassifiers * nbIter))
- temp_data = np.zeros((nbExamples, nbClassifiers))
- classifiersNames = multiclassResult.keys()
- for classifierIndex, (classifierName, errorOnExamplesDict) in enumerate(multiclassResult.items()):
- for iterIndex in range(nbIter):
- data[:, classifierIndex * nbIter + iterIndex] = errorOnExamplesDict["errorOnExample"]
- temp_data[:,classifierIndex] = errorOnExamplesDict["errorOnExample"]
- figWidth = max(nbClassifiers/2, minSize)
- figHeight = max(nbExamples/20, minSize)
- figKW = {"figsize":(figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- cmap = mpl.colors.ListedColormap(['red', 'green'])
- bounds = [-0.5, 0.5, 1.5]
- norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
-
- cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter)
- labels = classifiersNames
- plt.xticks(ticks, labels, rotation="vertical")
- red_patch = mpatches.Patch(color='red', label='Classifier failed')
- green_patch = mpatches.Patch(color='green', label='Classifier succeded')
- plt.legend(handles=[red_patch, green_patch], bbox_to_anchor=(0,1.02,1,0.2), loc="lower left",mode="expand", borderaxespad=0, ncol=2)
- fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-error_analysis.png", bbox_inches="tight")
- plt.close()
- logging.debug("Done:\t Label analysis figure generation")
-
- logging.debug("Start:\t Error by example figure generation")
- errorOnExamples = -1*np.sum(data, axis=1)/nbIter+nbClassifiers
- np.savetxt(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-clf_errors_doubled.csv", data, delimiter=",")
- np.savetxt(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-example_errors.csv", temp_data, delimiter=",")
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-example_errors.png")
- plt.close()
- logging.debug("Done:\t Error by example figure generation")
+ logging.debug("Start:\t Multiclass Label analysis figure generation")
+
+ base_file_name = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-"
+
+ nbClassifiers, nbExamples, nCopies, classifiersNames, data, errorOnExamples = gen_error_data(multiclassResult,
+ base_file_name)
+
+ publish2Dplot(data, classifiersNames, nbClassifiers, nbExamples, nCopies, base_file_name)
+
+ publishErrorsBarPlot(errorOnExamples, nbClassifiers, nbExamples, base_file_name)
+
+ logging.debug("Done:\t Multiclass Label analysis figure generation")
def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamples, nbLabels, multiclassLabels,
@@ -504,6 +501,7 @@ def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamp
publishMulticlassExmapleErrors(multiclassResults, directories, benchmarkArgumentDictionaries[0]["args"].name)
return multiclassResults
+
def numpy_mean_and_std(scores_array):
return np.mean(scores_array, axis=1), np.std(scores_array, axis=1)
@@ -532,57 +530,30 @@ def publishIterBiclassMetricsScores(iterResults, directory, labelsDictionary, cl
-def iterCmap(statsIter):
- cmapList = ["red", "0.0"]
- for i in range(statsIter):
- cmapList.append(str(float((i+1))/statsIter))
- cmap = mpl.colors.ListedColormap(cmapList)
- bounds = [-100*statsIter-0.5, -0.5]
- for i in range(statsIter):
- bounds.append(i+0.5)
- bounds.append(statsIter+0.5)
- norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
- return cmap, norm
+def gen_error_dat_glob(combiResults, statsIter, base_file_name):
+ nbExamples = combiResults["errorOnExamples"].shape[1]
+ nbClassifiers = combiResults["errorOnExamples"].shape[0]
+ data = np.transpose(combiResults["errorOnExamples"])
+ errorOnExamples = -1 * np.sum(data, axis=1) + (nbClassifiers * statsIter)
+ np.savetxt(base_file_name + "clf_errors.csv", data, delimiter=",")
+ np.savetxt(base_file_name + "example_errors.csv", errorOnExamples, delimiter=",")
+ return nbExamples, nbClassifiers, data, errorOnExamples
def publishIterBiclassExampleErrors(iterResults, directory, labelsDictionary, classifiersDict, statsIter, minSize=10):
for labelsCombination, combiResults in iterResults.items():
- currentDirectory = directory+ labelsDictionary[int(labelsCombination[0])]+"-vs-"+labelsDictionary[int(labelsCombination[1])]+"/"
- reversedClassifiersDict = dict((value, key) for key, value in classifiersDict.items())
- classifiersNames = [reversedClassifiersDict[i] for i in range(len(classifiersDict))]
-
- logging.debug("Start:\t Global label analysis figure generation")
- nbExamples = combiResults["errorOnExamples"].shape[1]
- nbClassifiers = combiResults["errorOnExamples"].shape[0]
- figWidth = max(nbClassifiers / 2, minSize)
- figHeight = max(nbExamples / 20, minSize)
- figKW = {"figsize": (figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- data = np.transpose(combiResults["errorOnExamples"])
- cmap, norm = iterCmap(statsIter)
- cax = plt.imshow(data, interpolation='none', cmap=cmap, norm=norm, aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbClassifiers)
- plt.xticks(ticks, classifiersNames, rotation="vertical")
- cbar = fig.colorbar(cax, ticks=[-100*statsIter/2, 0, statsIter])
- cbar.ax.set_yticklabels(['Unseen', 'Always Wrong', 'Always Right'])
- fig.tight_layout()
- fig.savefig(currentDirectory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-error_analysis.png")
- plt.close()
- logging.debug("Done:\t Global label analysis figure generation")
-
- logging.debug("Start:\t Global error by example figure generation")
- errorOnExamples = -1 * np.sum(data, axis=1) + (nbClassifiers*statsIter)
- np.savetxt(currentDirectory + time.strftime("%Y%m%d-%H%M%S") + "-clf_errors.csv", data, delimiter=",")
- np.savetxt(currentDirectory + time.strftime("%Y%m%d-%H%M%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers*statsIter])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(currentDirectory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-example_errors.png")
- plt.close()
- logging.debug("Done:\t Global error by example figure generation")
+ base_file_name = directory + labelsDictionary[int(labelsCombination[0])]+"-vs-"+\
+ labelsDictionary[int(labelsCombination[1])]+"/" + time.strftime("%Y_%m_%d-%H_%M_%S") + "-"
+ classifiersNames = [classifierName for classifierName in classifiersDict.values()]
+ logging.debug("Start:\t Global biclass label analysis figure generation")
+
+ nbExamples, nbClassifiers, data, errorOnExamples = gen_error_dat_glob(combiResults, statsIter, base_file_name)
+
+ publish2Dplot(data, classifiersNames, nbClassifiers, nbExamples, 1, base_file_name, statsIter=statsIter)
+
+ publishErrorsBarPlot(errorOnExamples, nbClassifiers*statsIter, nbExamples, base_file_name)
+
+ logging.debug("Done:\t Global biclass label analysis figures generation")
def publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames, dataBaseName, directory, statsIter, minSize=10):
@@ -600,40 +571,18 @@ def publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames,
train_STDs=trainSTDs, test_STDs=testSTDs)
-
def publishIterMulticlassExampleErrors(iterMulticlassResults, directory, classifiersNames, statsIter, minSize=10):
- logging.debug("Start:\t Global label analysis figure generation")
- nbExamples = iterMulticlassResults["errorOnExamples"].shape[1]
- nbClassifiers = iterMulticlassResults["errorOnExamples"].shape[0]
- figWidth = max(nbClassifiers / 2, minSize)
- figHeight = max(nbExamples / 20, minSize)
- figKW = {"figsize": (figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- data = np.transpose(iterMulticlassResults["errorOnExamples"])
- cax = plt.imshow(-data, interpolation='none', cmap="Greys", aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbClassifiers)
- plt.xticks(ticks, classifiersNames, rotation="vertical")
- cbar = fig.colorbar(cax, ticks=[0, -statsIter])
- cbar.ax.set_yticklabels(['Always Wrong', 'Always Right'])
- fig.tight_layout()
- fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-error_analysis.png")
- plt.close()
- logging.debug("Done:\t Global label analysis figure generation")
+ logging.debug("Start:\t Global multiclass label analysis figures generation")
+ base_file_name = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-"
- logging.debug("Start:\t Global error by example figure generation")
- errorOnExamples = -1 * np.sum(data, axis=1) + (nbClassifiers*statsIter)
- np.savetxt(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-clf_errors.csv", data, delimiter=",")
- np.savetxt(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-example_errors.csv", errorOnExamples, delimiter=",")
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, errorOnExamples)
- plt.ylim([0,nbClassifiers*statsIter])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-example_errors.png")
- plt.close()
- logging.debug("Done:\t Global error by example figure generation")
+ nbExamples, nbClassifiers, data, errorOnExamples = gen_error_dat_glob(iterMulticlassResults, statsIter, base_file_name)
+
+ publish2Dplot(data, classifiersNames, nbClassifiers, nbExamples, 1, base_file_name, statsIter=statsIter)
+
+ publishErrorsBarPlot(errorOnExamples, nbClassifiers * statsIter, nbExamples, base_file_name)
+
+ logging.debug("Done:\t Global multiclass label analysis figures generation")
def analyzebiclassIter(biclassResults, metrics, statsIter, directory, labelsDictionary, dataBaseName, nbExamples):
@@ -666,7 +615,7 @@ def analyzebiclassIter(biclassResults, metrics, statsIter, directory, labelsDict
iterBiclassResults[labelsComination]["metricsScores"][metric[0]]["trainScores"][classifiersDict[classifierName], iterIndex] = trainScore
iterBiclassResults[labelsComination]["metricsScores"][metric[0]]["testScores"][classifiersDict[classifierName], iterIndex] = testScore
for classifierName, errorOnExample in results["exampleErrors"].items():
- iterBiclassResults[labelsComination]["errorOnExamples"][classifiersDict[classifierName], :] += errorOnExample
+ iterBiclassResults[labelsComination]["errorOnExamples"][classifiersDict[classifierName], :] += errorOnExample["errorOnExamples"]
publishIterBiclassMetricsScores(iterBiclassResults, directory, labelsDictionary, classifiersDict, dataBaseName, statsIter)
publishIterBiclassExampleErrors(iterBiclassResults, directory, labelsDictionary, classifiersDict, statsIter)
@@ -693,7 +642,7 @@ def analyzeIterMulticlass(multiclassResults, directory, statsIter, metrics, data
np.zeros((nbClassifiers, statsIter))}
iterMulticlassResults["metricsScores"][metric[0]]["trainScores"][classifierIndex, iterIndex] = classifierResults["metricsScores"][metric[0]][0]
iterMulticlassResults["metricsScores"][metric[0]]["testScores"][classifierIndex, iterIndex] = classifierResults["metricsScores"][metric[0]][1]
- iterMulticlassResults["errorOnExamples"][classifierIndex, :] += classifierResults["errorOnExample"]
+ iterMulticlassResults["errorOnExamples"][classifierIndex, :] += classifierResults["errorOnExamples"]
logging.debug("Start:\t Getting mean results for multiclass classification")
classifiersNames = np.array(classifiersNames)
--
GitLab