diff --git a/Code/MonoMultiViewClassifiers/ExecClassif.py b/Code/MonoMultiViewClassifiers/ExecClassif.py index d399b6216ec3398622899da0bd1ed6568f8f37bd..b5360978e6060743ff25ca7362db294e919daf9d 100644 --- a/Code/MonoMultiViewClassifiers/ExecClassif.py +++ b/Code/MonoMultiViewClassifiers/ExecClassif.py @@ -377,7 +377,7 @@ def execClassif(arguments): metrics = [metric.split(":") for metric in args.CL_metrics] if metrics == [[""]]: metricsNames = [name for _, name, isPackage - in pkgutil.iter_modules(['./MonoMultiViewClassifiers/Metrics']) if not isPackage and name != "log_loss"] + in pkgutil.iter_modules(['./MonoMultiViewClassifiers/Metrics']) if not isPackage and name not in ["log_loss", "matthews_corrcoef", "roc_auc_score"]] metrics = [[metricName] for metricName in metricsNames] metrics = arangeMetrics(metrics, args.CL_metric_princ) for metricIndex, metric in enumerate(metrics): diff --git a/Code/MonoMultiViewClassifiers/Metrics/f1_score.py b/Code/MonoMultiViewClassifiers/Metrics/f1_score.py index 3a6517a1a0f5872c7d95944c091c1a1896dcae3a..0a4de7a751a6319ae9d399c6734601061e69a56a 100644 --- a/Code/MonoMultiViewClassifiers/Metrics/f1_score.py +++ b/Code/MonoMultiViewClassifiers/Metrics/f1_score.py @@ -27,7 +27,7 @@ def score(y_true, y_pred, **kwargs): try: average = kwargs["3"] except: - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: average = "micro" else: average = "binary" diff --git a/Code/MonoMultiViewClassifiers/Metrics/fbeta_score.py b/Code/MonoMultiViewClassifiers/Metrics/fbeta_score.py index 84e3f85cf75565cc60f6179ca253e55b4db1d884..014e5fbb5b8af2f4602fc479c00b317aff65e0e9 100644 --- a/Code/MonoMultiViewClassifiers/Metrics/fbeta_score.py +++ b/Code/MonoMultiViewClassifiers/Metrics/fbeta_score.py @@ -26,7 +26,7 @@ def score(y_true, y_pred, **kwargs): try: average = kwargs["4"] except: - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: average = "micro" else: average = "binary" diff --git a/Code/MonoMultiViewClassifiers/Metrics/precision_score.py b/Code/MonoMultiViewClassifiers/Metrics/precision_score.py index a5bbd477ddeb25dfd43a8609e10c1aa65e87190a..c72ddd2965729f86bdf1d2db57f37092eeeea31e 100644 --- a/Code/MonoMultiViewClassifiers/Metrics/precision_score.py +++ b/Code/MonoMultiViewClassifiers/Metrics/precision_score.py @@ -22,7 +22,7 @@ def score(y_true, y_pred, **kwargs): try: average = kwargs["3"] except: - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: average = "micro" else: average = "binary" diff --git a/Code/MonoMultiViewClassifiers/Metrics/recall_score.py b/Code/MonoMultiViewClassifiers/Metrics/recall_score.py index b1e3baa493d1ff61875c56f5588f67c0fb19fd05..925169fcec2cf055886cdd8dbf752a7cb7693278 100644 --- a/Code/MonoMultiViewClassifiers/Metrics/recall_score.py +++ b/Code/MonoMultiViewClassifiers/Metrics/recall_score.py @@ -22,7 +22,7 @@ def score(y_true, y_pred, **kwargs): try: average = kwargs["3"] except: - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: average = "micro" else: average = "binary" diff --git a/Code/MonoMultiViewClassifiers/Metrics/roc_auc_score.py b/Code/MonoMultiViewClassifiers/Metrics/roc_auc_score.py index 2f31f24f75f526a11404d4312bbb99def07727c4..03d21c06caa867d80aee8cb66a14ddb1370e1527 100644 --- a/Code/MonoMultiViewClassifiers/Metrics/roc_auc_score.py +++ b/Code/MonoMultiViewClassifiers/Metrics/roc_auc_score.py @@ -15,11 +15,11 @@ def score(y_true, y_pred, **kwargs): try: average = kwargs["1"] except: - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: average = "micro" else: average = None - if len(set(y_true)) > 2: + if len(set(y_true)) > 2 or len(set(y_pred))>2: mlb = MultiLabelBinarizer() y_true = mlb.fit_transform([(label) for label in y_true]) y_pred = mlb.fit_transform([(label) for label in y_pred]) diff --git a/Code/MonoMultiViewClassifiers/MultiviewClassifiers/Fusion/FusionModule.py b/Code/MonoMultiViewClassifiers/MultiviewClassifiers/Fusion/FusionModule.py index ab5ceedd276f23c6d13a3a3b801318cb9e0a4cac..b8b74cb54480ad247ec0f1595b0ad9e3dc1d93b4 100644 --- a/Code/MonoMultiViewClassifiers/MultiviewClassifiers/Fusion/FusionModule.py +++ b/Code/MonoMultiViewClassifiers/MultiviewClassifiers/Fusion/FusionModule.py @@ -153,7 +153,7 @@ def getCLString(classificationKWARGS): """Used to get the classifier name as a string""" if classificationKWARGS["fusionType"] == "LateFusion": return "Fusion-" + classificationKWARGS["fusionType"] + "-" + classificationKWARGS["fusionMethod"] + "-" + \ - "-".join(classificationKWARGS["classifiersNames"]) + "-".join(classificationKWARGS["classifiersNames"][:5]) elif classificationKWARGS["fusionType"] == "EarlyFusion": return "Fusion-" + classificationKWARGS["fusionType"] + "-" + classificationKWARGS["fusionMethod"] + "-" + \ classificationKWARGS["classifiersNames"] diff --git a/Code/MonoMultiViewClassifiers/ResultAnalysis.py b/Code/MonoMultiViewClassifiers/ResultAnalysis.py index 5017f24459916e5e5c37598dad8b897a1a9ac785..f1c92fe6b449e289e266f057e032c21fe00c7dcf 100644 --- a/Code/MonoMultiViewClassifiers/ResultAnalysis.py +++ b/Code/MonoMultiViewClassifiers/ResultAnalysis.py @@ -8,6 +8,7 @@ import matplotlib # matplotlib.use('Agg') import matplotlib.pyplot as plt +import matplotlib.patches as mpatches import numpy as np # from matplotlib import cm import matplotlib as mpl @@ -21,13 +22,17 @@ __author__ = "Baptiste Bauvin" __status__ = "Prototype" # Production, Development, Prototype -def autolabel(rects, ax): +def autolabel(rects, ax, set=1): """Used to print scores on top of the bars""" + if set == 1: + text_height = -0.05 + else: + text_height = -0.07 for rect in rects: height = rect.get_height() - ax.text(rect.get_x() + rect.get_width() / 2., 1.01 * height, + ax.text(rect.get_x() + rect.get_width() / 2., text_height, "%.2f" % height, - ha='center', va='bottom') + ha='center', va='bottom', size="small") def getMetricsScoresBiclass(metrics, monoviewResults, multiviewResults): @@ -39,7 +44,7 @@ def getMetricsScoresBiclass(metrics, monoviewResults, multiviewResults): for classifierResult in monoviewResults: trainScores.append(classifierResult[1][2][metric[0]][0]) testScores.append(classifierResult[1][2][metric[0]][1]) - classifiersNames.append(classifierResult[1][0]) + classifiersNames.append(classifierResult[1][0]+"-"+classifierResult[1][1][-1]) for classifierResult in multiviewResults: trainScores.append(classifierResult[2][metric[0]][0]) testScores.append(classifierResult[2][metric[0]][1]) @@ -56,7 +61,7 @@ def getExampleErrorsBiclass(usedBenchmarkArgumentDictionary, monoviewResults, mu exampleErrors = {} trueLabels = usedBenchmarkArgumentDictionary["labels"] for classifierResult in monoviewResults: - classifierName = classifierResult[1][0] + classifierName = classifierResult[1][0]+"-"+classifierResult[1][1][-1] predictedLabels = classifierResult[1][3] errorOnExamples = predictedLabels==trueLabels errorOnExamples = errorOnExamples.astype(int) @@ -90,7 +95,7 @@ def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames,min size = nbResults if nbResults < minSize: size = minSize - figKW = {"figsize" : (size, 3.0/4*size+2.0)} + figKW = {"figsize" : (size, size/5)} f, ax = plt.subplots(nrows=1, ncols=1, **figKW) barWidth= 0.35 sorted_indices = np.argsort(testScores) @@ -101,8 +106,8 @@ def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames,min ax.set_title(metricName + "\n scores for each classifier") rects = ax.bar(range(nbResults), testScores, barWidth, color="r", ) rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", ) - autolabel(rects, ax) - autolabel(rect2, ax) + autolabel(rects, ax, set=1) + autolabel(rect2, ax, set=2) ax.legend((rects[0], rect2[0]), ('Test', 'Train')) ax.set_ylim(-0.1, 1.1) ax.set_xticks(np.arange(nbResults) + barWidth) @@ -138,8 +143,12 @@ def publishExampleErrors(exampleErrors, directory, databaseName, labelsNames,min ticks = np.arange(nbIter/2-0.5, nbClassifiers * nbIter, nbIter) labels = classifiersNames plt.xticks(ticks, labels, rotation="vertical") - cbar = fig.colorbar(cax, ticks=[0, 1]) - cbar.ax.set_yticklabels(['Unseen', 'Wrong', ' Right']) + red_patch = mpatches.Patch(color='red', label='Classifier failed') + green_patch = mpatches.Patch(color='green', label='Classifier succeded') + black_patch = mpatches.Patch(color='black', label='Unseen data') + plt.legend(handles=[red_patch, green_patch, black_patch], bbox_to_anchor=(1.04, 1)) + # cbar = fig.colorbar(cax, ticks=[0, 1, -100]) + # cbar.ax.set_yticklabels(['Unseen', 'Wrong', ' Right']) fig.tight_layout() fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-"+"vs".join(labelsNames)+ "-error_analysis.png") plt.close() @@ -259,8 +268,8 @@ def publishMulticlassScores(multiclassResults, metrics, statsIter, direcories, d ax.set_title(metric[0] + "\n on validation set for each classifier") rects = ax.bar(range(nbResults), validationScores, barWidth, color="r", ) rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", ) - autolabel(rects, ax) - autolabel(rect2, ax) + autolabel(rects, ax, set=1) + autolabel(rect2, ax, set=2) ax.legend((rects[0], rect2[0]), ('Test', 'Train')) ax.set_ylim(-0.1, 1.1) ax.set_xticks(np.arange(nbResults) + barWidth) @@ -271,8 +280,8 @@ def publishMulticlassScores(multiclassResults, metrics, statsIter, direcories, d logging.debug("Done:\t Multiclass score graph generation for " + metric[0]) -def publishMulticlassExmapleErrors(multiclassResults, directories, databaseName, labelsNames, minSize=10): - for iterIndex, multiclassResult in multiclassResults: +def publishMulticlassExmapleErrors(multiclassResults, directories, databaseName, minSize=10): + for iterIndex, multiclassResult in enumerate(multiclassResults): directory = directories[iterIndex] logging.debug("Start:\t Label analysis figure generation") nbClassifiers = len(multiclassResult) @@ -314,14 +323,14 @@ def publishMulticlassExmapleErrors(multiclassResults, directories, databaseName, plt.bar(x, errorOnExamples) plt.ylim([0,nbClassifiers]) plt.title("Number of classifiers that failed to classify each example") - fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-"+"vs".join(labelsNames)+ "-example_errors.png") + fig.savefig(directory + time.strftime("%Y%m%d-%H%M%S") + "-" + databaseName +"-example_errors.png") plt.close() logging.debug("Done:\t Error by example figure generation") def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamples, nbLabels, multiclassLabels, metrics, classificationIndices, directories): - """Used to tranform one versus one results in multiclass results and to publish it""" + """Used to transform one versus one results in multiclass results and to publish it""" multiclassResults = [{} for _ in range(statsIter)] for flag, resMono, resMulti in results: iterIndex = flag[0] @@ -331,20 +340,21 @@ def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamp if benchmarkArgumentDictionary["flag"] == flag: trainIndices, testIndices, testMulticlassIndices = benchmarkArgumentDictionary["classificationIndices"] for classifierResult in resMono: - if classifierResult[1][0] not in multiclassResults[iterIndex]: - multiclassResults[iterIndex][classifierResult[1][0]] = np.zeros((nbExamples, nbLabels),dtype=int) + classifierName = classifierResult[1][0]+"-"+classifierResult[1][1][-1] + if classifierName not in multiclassResults[iterIndex]: + multiclassResults[iterIndex][classifierName] = np.zeros((nbExamples, nbLabels),dtype=int) for exampleIndex in trainIndices: label = classifierResult[1][3][exampleIndex] if label == 1: - multiclassResults[iterIndex][classifierResult[1][0]][exampleIndex, classifierPositive] += 1 + multiclassResults[iterIndex][classifierName][exampleIndex, classifierPositive] += 1 else: - multiclassResults[iterIndex][classifierResult[1][0]][exampleIndex, classifierNegative] += 1 + multiclassResults[iterIndex][classifierName][exampleIndex, classifierNegative] += 1 for multiclassIndex, exampleIndex in enumerate(testMulticlassIndices): label = classifierResult[1][5][multiclassIndex] if label == 1: - multiclassResults[iterIndex][classifierResult[1][0]][exampleIndex, classifierPositive] += 1 + multiclassResults[iterIndex][classifierName][exampleIndex, classifierPositive] += 1 else: - multiclassResults[iterIndex][classifierResult[1][0]][exampleIndex, classifierNegative] += 1 + multiclassResults[iterIndex][classifierName][exampleIndex, classifierNegative] += 1 for classifierResult in resMulti: multiviewClassifierPackage = getattr(MultiviewClassifiers, classifierResult[0]) @@ -379,7 +389,7 @@ def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamp multiclassResults = getErrorOnLabelsMulticlass(multiclassResults, multiclassLabels) publishMulticlassScores(multiclassResults, metrics, statsIter, directories, benchmarkArgumentDictionaries[0]["args"].name) - publishMulticlassExmapleErrors(multiclassResults, metrics, statsIter, directories, benchmarkArgumentDictionaries[0]["args"].name) + publishMulticlassExmapleErrors(multiclassResults, directories, benchmarkArgumentDictionaries[0]["args"].name) return multiclassResults @@ -411,8 +421,8 @@ def publishIterBiclassMetricsScores(iterResults, directory, labelsDictionary, cl ax.set_title(metricName + " for each classifier") rects = ax.bar(range(nbResults), testMeans, barWidth, color="r", yerr=testSTDs) rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs) - autolabel(rects, ax) - autolabel(rect2, ax) + autolabel(rects, ax, set=1) + autolabel(rect2, ax, set=2) ax.set_ylim(-0.1, 1.1) ax.legend((rects[0], rect2[0]), ('Test', 'Train')) ax.set_xticks(np.arange(nbResults) + barWidth) @@ -486,8 +496,8 @@ def publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames, ax.set_title(metricName + " for each classifier") rects = ax.bar(range(nbResults), testMeans, barWidth, color="r", yerr=testSTDs) rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs) - autolabel(rects, ax) - autolabel(rect2, ax) + autolabel(rects, ax, set=1) + autolabel(rect2, ax, set=2) ax.set_ylim(-0.1, 1.1) ax.legend((rects[0], rect2[0]), ('Test', 'Train')) ax.set_xticks(np.arange(nbResults) + barWidth) @@ -535,7 +545,7 @@ def publishIterMulticlassExampleErrors(iterMulticlassResults, directory, classif def analyzebiclassIter(biclassResults, metrics, statsIter, directory, labelsDictionary, dataBaseName): iterBiclassResults = {} classifiersDict = {} - for iterIndex, biclassResult in biclassResults: + for iterIndex, biclassResult in enumerate(biclassResults): for labelsComination, results in biclassResult.items(): for metric in metrics: nbClassifiers = len(results["metricsScores"][metric[0]]["classifiersNames"]) @@ -552,16 +562,16 @@ def analyzebiclassIter(biclassResults, metrics, statsIter, directory, labelsDict dtype=int) if metric[0] not in iterBiclassResults[labelsComination]["metricsScores"]: iterBiclassResults[labelsComination]["metricsScores"][metric[0]]= {"trainScores": - np.zeros(nbClassifiers, statsIter), + np.zeros((nbClassifiers, statsIter)), "testScores": - np.zeros(nbClassifiers, statsIter)} + np.zeros((nbClassifiers, statsIter))} for classifierName, trainScore, testScore in zip(results["metricsScores"][metric[0]]["classifiersNames"], results["metricsScores"][metric[0]]["trainScores"], results["metricsScores"][metric[0]]["testScores"], ): iterBiclassResults[labelsComination]["metricsScores"][metric[0]]["trainScores"][classifiersDict[classifierName], iterIndex] = trainScore iterBiclassResults[labelsComination]["metricsScores"][metric[0]]["testScores"][classifiersDict[classifierName], iterIndex] = testScore - for classifierName, errorOnExample in results["errorOnExamples"]: + for classifierName, errorOnExample in results["exampleErrors"].items(): iterBiclassResults[labelsComination]["errorOnExamples"][classifiersDict[classifierName], :] += errorOnExample publishIterBiclassMetricsScores(iterBiclassResults, directory, labelsDictionary, classifiersDict, dataBaseName, statsIter) publishIterBiclassExampleErrors(iterBiclassResults, directory, labelsDictionary, classifiersDict, statsIter) @@ -577,19 +587,19 @@ def analyzeIterMulticlass(multiclassResults, directory, statsIter, metrics, data dtype=int) iterMulticlassResults["metricsScores"] = {} classifiersNames = [] - for iterIndex, multiclassResult in multiclassResults: + for iterIndex, multiclassResult in enumerate(multiclassResults): for classifierName, classifierResults in multiclassResult.items(): classifiersNames.append(classifierName) classifierIndex = len(classifiersNames)-1 for metric in metrics: if metric[0] not in iterMulticlassResults["metricsScores"]: iterMulticlassResults["metricsScores"][metric[0]] = {"trainScores": - np.zeros(nbClassifiers, statsIter), + np.zeros((nbClassifiers, statsIter)), "testScores": - np.zeros(nbClassifiers, statsIter)} + np.zeros((nbClassifiers, statsIter))} iterMulticlassResults["metricsScores"][metric[0]]["trainScores"][classifierIndex, iterIndex] = classifierResults["metricsScores"][metric[0]][0] iterMulticlassResults["metricsScores"][metric[0]]["testScores"][classifierIndex, iterIndex] = classifierResults["metricsScores"][metric[0]][1] - iterMulticlassResults["errorOnExamples"][classifierIndex, :] = classifierResults["exampleErrors"] + iterMulticlassResults["errorOnExamples"][classifierIndex, :] += classifierResults["exampleErrors"] logging.debug("Start:\t Getting mean results for multiclass classification") publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames, dataBaseName, directory, statsIter)