diff --git a/multiview_platform/MonoMultiViewClassifiers/ExecClassif.py b/multiview_platform/MonoMultiViewClassifiers/ExecClassif.py index 80ac9dc5c9066f57ca84592581eaefb1e2db4f71..a75cd6e4cbce7627ec13bf0dc3845db1e0876349 100644 --- a/multiview_platform/MonoMultiViewClassifiers/ExecClassif.py +++ b/multiview_platform/MonoMultiViewClassifiers/ExecClassif.py @@ -278,7 +278,7 @@ def execOneBenchmark(coreIndex=-1, LABELS_DICTIONARY=None, directory=None, class for arguments in argumentDictionaries["Multiview"]] logging.debug("Done:\t Multiview benchmark") - return [flag, resultsMonoview, resultsMultiview] + return [flag, resultsMonoview + resultsMultiview] def execOneBenchmark_multicore(nbCores=-1, LABELS_DICTIONARY=None, directory=None, classificationIndices=None, args=None, @@ -323,7 +323,7 @@ def execOneBenchmark_multicore(nbCores=-1, LABELS_DICTIONARY=None, directory=Non for coreIndex in range(min(nbCores, nbExperiments - stepIndex * nbCores))) logging.debug("Done:\t Multiview benchmark") - return [flag, resultsMonoview, resultsMultiview] + return [flag, resultsMonoview + resultsMultiview] def execOneBenchmarkMonoCore(DATASET=None, LABELS_DICTIONARY=None, directory=None, classificationIndices=None, args=None, diff --git a/multiview_platform/MonoMultiViewClassifiers/Multiview/Additions/diversity_utils.py b/multiview_platform/MonoMultiViewClassifiers/Multiview/Additions/diversity_utils.py index 84f8c269d95138ac0fa0886342fb26b9897dcdc4..59a35690cc108c1ff39ece36232247c1683e7857 100644 --- a/multiview_platform/MonoMultiViewClassifiers/Multiview/Additions/diversity_utils.py +++ b/multiview_platform/MonoMultiViewClassifiers/Multiview/Additions/diversity_utils.py @@ -147,7 +147,7 @@ def getFoldsGroundTruth(directory, folds=True): foldsGroudTruth = np.zeros((len(foldsFilesNames), foldLen), dtype=int) for fileName in foldsFilesNames: foldIndex = int(fileName[-5]) - foldsGroudTruth[foldIndex] = np.genfromtxt(directory+"folds/"+fileName, delimiter=',') + foldsGroudTruth[foldIndex] = np.genfromtxt(directory+"folds/"+fileName, delimiter=',')[:foldLen] return foldsGroudTruth else: train_labels = np.genfromtxt(directory+"train_labels.csv", delimiter=',') diff --git a/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py b/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py index 8905d16b014c47940a2a8b10397cc58281672ceb..2ad258ddafa123450b94fa07bde8c48ba109a2e7 100644 --- a/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py +++ b/multiview_platform/MonoMultiViewClassifiers/ResultAnalysis.py @@ -62,22 +62,22 @@ def autolabel(rects, ax, set=1, std=None): def getMetricsScoresBiclass(metrics, results): r"""Used to extract metrics scores in case of biclass classification - Parameters - ---------- - metrics : list of lists - The metrics names with configuration metrics[i][0] = name of metric i - results : list of MonoviewResult and MultiviewResults objects - A list containing all the resluts for all the monoview experimentations. - - Returns - ------- - metricsScores : dict of dict of list - Regroups all the scores for each metrics for each classifier and for the train and test sets. - organized as : - -`metricScores[metric_name]["classifiersNames"]` is a list of all the classifiers available for this metric, - -`metricScores[metric_name]["trainScores"]` is a list of all the available classifiers scores on the train set, - -`metricScores[metric_name]["testScores"]` is a list of all the available classifiers scores on the test set. - """ + Parameters + ---------- + metrics : list of lists + The metrics names with configuration metrics[i][0] = name of metric i + results : list of MonoviewResult and MultiviewResults objects + A list containing all the resluts for all the monoview experimentations. + + Returns + ------- + metricsScores : dict of dict of list + Regroups all the scores for each metrics for each classifier and for the train and test sets. + organized as : + -`metricScores[metric_name]["classifiersNames"]` is a list of all the classifiers available for this metric, + -`metricScores[metric_name]["trainScores"]` is a list of all the available classifiers scores on the train set, + -`metricScores[metric_name]["testScores"]` is a list of all the available classifiers scores on the test set. + """ metricsScores = {} for metric in metrics: @@ -96,60 +96,181 @@ def getMetricsScoresBiclass(metrics, results): return metricsScores -def getExampleErrorsBiclass(usedBenchmarkArgumentDictionary, results): +def getExampleErrorsBiclass(groud_truth, results): + r"""Used to get for each classifier and each example whether the classifier has misclassified the example or not. + + Parameters + ---------- + ground_truth : numpy array of 0, 1 and -100 (if multiclass) + The array with the real labels of the examples + results : list of MonoviewResult and MultiviewResults objects + A list containing all the resluts for all the mono- & multi-view experimentations. + + Returns + ------- + exampleErrors : dict of np.array + For each classifier, has an entry with a `np.array` over the examples, with a 1 if the examples was + well-classified, a 0 if not and if it's multiclass classification, a -100 if the examples was not seen during + the one versus one classification. + """ exampleErrors = {} - trueLabels = usedBenchmarkArgumentDictionary["labels"] for classifierResult in results: - classifierName = classifierResult.get_classifier_name() - predictedLabels = classifierResult.full_labels_pred - errorOnExamples = predictedLabels==trueLabels - errorOnExamples = errorOnExamples.astype(int) - unseenExamples = np.where(trueLabels==-100)[0] + errorOnExamples = np.equal(classifierResult.full_labels_pred, groud_truth).astype(int) + unseenExamples = np.where(groud_truth==-100)[0] errorOnExamples[unseenExamples]=-100 - exampleErrors[classifierName] = errorOnExamples + exampleErrors[classifierResult.get_classifier_name()] = errorOnExamples return exampleErrors -def plotMetricOneIter(trainScores, testScores, names, nbResults, metricName, fileName, minSize=15): - testScores = np.array(testScores) - trainScores = np.array(trainScores) - names = np.array(names) - size = nbResults - if nbResults < minSize: - size = minSize - figKW = {"figsize" : (size, size/3)} +def get_fig_size(nb_results, min_size=15, multiplier=1.0, bar_width=0.35): + r"""Used to get the image size to save the figure and the bar width, depending on the number of scores to plot. + + Parameters + ---------- + nb_results : int + The number of couple of bar to plot. + min_size : int + The minimum size of the image, if there are few classifiers to plot. + multiplier : float + The ratio between the image size and the number of classifiers. + bar_width : float + The width of the bars in the figure. Mainly here to centralize bar_width. + + Returns + ------- + fig_kwargs : dict of arguments + The argument restraining the size of the figure, usable directly in the `subplots` function of + `matplotlib.pyplot`. + bar_width : float + The width of the bars in the figure. Mainly here to centralize bar_width. + """ + size = nb_results*multiplier + if size < min_size: + size = min_size + fig_kwargs = {"figsize": (size, size / 3)} + return fig_kwargs, bar_width + + +def sort_by_test_score(train_scores,test_scores, names, train_STDs=None, test_STDs=None): + r"""Used to sort the results (names and both scores) in descending test score order. + + Parameters + ---------- + train_scores : np.array of floats + The scores of each classifier on the training set. + test_scores : np.array of floats + The scores of each classifier on the testing set. + names : np.array of strs + The names of all the classifiers. + train_STDs : np.array of floats or None + The array containing the standard deviations for the averaged scores on the training set. + test_STDs : np.array of floats or None + The array containing the standard deviations for the averaged scores on the testing set. + + Returns + ------- + sorted_names : np.array of strs + The names of all the classifiers, sorted in descending test score order. + sorted_train_scores : np.array of floats + The scores of each classifier on the training set, sorted in descending test score order. + sorted_test_scores : np.array of floats + The scores of each classifier on the testing set, sorted in descending test score order. + sorted_train_STDs : np.array of floats or None + The array containing the standard deviations for the averaged scores on the training set, + sorted in descending test score order. + sorted_test_STDs : np.array of floats or None + The array containing the standard deviations for the averaged scores on the testing set, + sorted in descending test score order. + """ + sorted_indices = np.argsort(test_scores) + sorted_test_scores = test_scores[sorted_indices] + sorted_train_scores = train_scores[sorted_indices] + sorted_names = names[sorted_indices] + if train_STDs is not None and test_STDs is not None: + sorted_train_STDs = train_STDs[sorted_indices] + sorted_test_STDs = test_STDs[sorted_indices] + else: + sorted_train_STDs = None + sorted_test_STDs = None + return sorted_names, sorted_train_scores, sorted_test_scores, sorted_train_STDs, sorted_test_STDs + + +def plotMetricScores(trainScores, testScores, names, nbResults, metricName, fileName, + tag="", train_STDs=None, test_STDs=None): + r"""Used to plot and save the score barplot for a specific metric. + + Parameters + ---------- + trainScores : list or np.array of floats + The scores of each classifier on the training set. + testScores : list or np.array of floats + The scores of each classifier on the testing set. + names : list or np.array of strs + The names of all the classifiers. + nbResults: int + The number of classifiers to plot. + metricName : str + The plotted metric's name + fileName : str + The name of the file where the figure will be saved. + + Returns + ------- + """ + + figKW, barWidth = get_fig_size(nbResults) + + names, trainScores, testScores, train_STDs, test_STDs = sort_by_test_score(trainScores, testScores, names, + train_STDs, test_STDs) + f, ax = plt.subplots(nrows=1, ncols=1, **figKW) - barWidth= 0.35 - sorted_indices = np.argsort(testScores) - testScores = testScores[sorted_indices] - trainScores = trainScores[sorted_indices] - names = names[sorted_indices] - - ax.set_title(metricName + "\n scores for each classifier") - rects = ax.bar(range(nbResults), testScores, barWidth, color="r", ) - rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", ) - autolabel(rects, ax, set=1) - autolabel(rect2, ax, set=2) + ax.set_title(metricName + "\n"+ tag +" scores for each classifier") + + rects = ax.bar(range(nbResults), testScores, barWidth, color="0.8", yerr=test_STDs) + rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.5", yerr=train_STDs) + autolabel(rects, ax, set=1, std=test_STDs) + autolabel(rect2, ax, set=2, std=train_STDs) + ax.legend((rects[0], rect2[0]), ('Test', 'Train')) ax.set_ylim(-0.1, 1.1) ax.set_xticks(np.arange(nbResults) + barWidth) ax.set_xticklabels(names, rotation="vertical") + plt.tight_layout() f.savefig(fileName) plt.close() def publishMetricsGraphs(metricsScores, directory, databaseName, labelsNames): + r"""Used to sort the results (names and both scores) in descending test score order. + + Parameters + ---------- + metricsScores : dict of dicts of lists or np.arrays + Keys : The names of the metrics. + Values : The scores and names of each classifier . + directory : str + The path to the directory where the figures will be saved. + databaseName : str + The name of the database on which the experiments where conducted. + labelsNames : list of strs + The name corresponding to each numerical label. + + Returns + ------- + """ for metricName, metricScores in metricsScores.items(): logging.debug("Start:\t Biclass score graph generation for "+metricName) - trainScores = metricScores["trainScores"] - testScores = metricScores["testScores"] - names = metricScores["classifiersNames"] - nbResults = len(testScores) + + nbResults = len(metricScores["testScores"]) + fileName = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName +"-"+"_vs_".join(labelsNames)+ "-" + metricName + ".png" - plotMetricOneIter(trainScores, testScores, names, nbResults, metricName, fileName) + + plotMetricScores(np.array(metricScores["trainScores"]), np.array(metricScores["testScores"]), + np.array(metricScores["classifiersNames"]), nbResults, metricName, fileName, tag=" "+" vs ".join(labelsNames)) + logging.debug("Done:\t Biclass score graph generation for " + metricName) @@ -220,7 +341,7 @@ def analyzeBiclass(results, benchmarkArgumentDictionaries, statsIter, metrics): if benchmarkArgumentDictionary["flag"]==flag: usedBenchmarkArgumentDictionary = benchmarkArgumentDictionary metricsScores = getMetricsScoresBiclass(metrics, result) - exampleErrors = getExampleErrorsBiclass(usedBenchmarkArgumentDictionary, result) + exampleErrors = getExampleErrorsBiclass(usedBenchmarkArgumentDictionary["labels"], result) directory = usedBenchmarkArgumentDictionary["directory"] databaseName = usedBenchmarkArgumentDictionary["args"].name labelsNames = [usedBenchmarkArgumentDictionary["LABELS_DICTIONARY"][0], @@ -277,41 +398,18 @@ def publishMulticlassScores(multiclassResults, metrics, statsIter, direcories, d directory = direcories[iterIndex] for metric in metrics: logging.debug("Start:\t Multiclass score graph generation for "+metric[0]) - classifiersNames = [] - validationScores = [] - trainScores = [] - for classifierName in multiclassResults[iterIndex].keys(): - classifiersNames.append(classifierName) - validationScores.append(multiclassResults[iterIndex][classifierName]["metricsScores"][metric[0]][1]) - trainScores.append(multiclassResults[iterIndex][classifierName]["metricsScores"][metric[0]][0]) - nbResults = len(validationScores) - - validationScores = np.array(validationScores) - trainScores = np.array(trainScores) - names = np.array(classifiersNames) - size = nbResults - if nbResults < minSize: - size = minSize - figKW = {"figsize" : (size, size/3)} - f, ax = plt.subplots(nrows=1, ncols=1, **figKW) - barWidth= 0.35 - sorted_indices = np.argsort(validationScores) - validationScores = validationScores[sorted_indices] - trainScores = trainScores[sorted_indices] - names = names[sorted_indices] - - ax.set_title(metric[0] + "\n on validation set for each classifier") - rects = ax.bar(range(nbResults), validationScores, barWidth, color="r", ) - rect2 = ax.bar(np.arange(nbResults) + barWidth, trainScores, barWidth, color="0.7", ) - autolabel(rects, ax, set=1) - autolabel(rect2, ax, set=2) - ax.legend((rects[0], rect2[0]), ('Test', 'Train')) - ax.set_ylim(-0.1, 1.1) - ax.set_xticks(np.arange(nbResults) + barWidth) - ax.set_xticklabels(names, rotation="vertical") - plt.tight_layout() - f.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName + "-" + metric[0] + ".png") - plt.close() + classifiersNames = np.array([classifierName for classifierName in multiclassResults[iterIndex].keys()]) + trainScores = np.array([multiclassResults[iterIndex][classifierName]["metricsScores"][metric[0]][0] + for classifierName in classifiersNames]) + validationScores = np.array([multiclassResults[iterIndex][classifierName]["metricsScores"][metric[0]][1] + for classifierName in classifiersNames]) + + + nbResults = classifiersNames.shape[0] + fileName = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + databaseName + "-" + metric[0] + ".png" + + plotMetricScores(trainScores, validationScores, classifiersNames, nbResults, metric[0], fileName, tag=" multiclass") + logging.debug("Done:\t Multiclass score graph generation for " + metric[0]) @@ -406,6 +504,9 @@ def analyzeMulticlass(results, statsIter, benchmarkArgumentDictionaries, nbExamp publishMulticlassExmapleErrors(multiclassResults, directories, benchmarkArgumentDictionaries[0]["args"].name) return multiclassResults +def numpy_mean_and_std(scores_array): + return np.mean(scores_array, axis=1), np.std(scores_array, axis=1) + def publishIterBiclassMetricsScores(iterResults, directory, labelsDictionary, classifiersDict, dataBaseName, statsIter, minSize=10): for labelsCombination, iterResult in iterResults.items(): @@ -416,43 +517,19 @@ def publishIterBiclassMetricsScores(iterResults, directory, labelsDictionary, cl except OSError as exc: if exc.errno != errno.EEXIST: raise + for metricName, scores in iterResult["metricsScores"].items(): - trainScores = scores["trainScores"] - testScores = scores["testScores"] - trainMeans = np.mean(trainScores, axis=1) - testMeans = np.mean(testScores, axis=1) - trainSTDs = np.std(trainScores, axis=1) - testSTDs = np.std(testScores, axis=1) - nbResults = len(trainMeans) - reversedClassifiersDict = dict((value, key) for key, value in classifiersDict.items()) - # import pdb;pdb.set_trace() - names = [reversedClassifiersDict[i] for i in range(len(classifiersDict))] - size=nbResults - if nbResults<minSize: - size=minSize - figKW = {"figsize" : (size, size/3)} - f, ax = plt.subplots(nrows=1, ncols=1, **figKW) - barWidth = 0.35 - sorted_indices = np.argsort(testMeans) - testMeans = testMeans[sorted_indices] - testSTDs = testSTDs[sorted_indices] - trainSTDs = trainSTDs[sorted_indices] - trainMeans = trainMeans[sorted_indices] - names = np.array(names)[sorted_indices] - - ax.set_title(metricName + " for each classifier") - rects = ax.bar(range(nbResults), testMeans, barWidth, color="r", yerr=testSTDs) - rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs) - autolabel(rects, ax, set=1, std=testSTDs) - autolabel(rect2, ax, set=2, std=trainSTDs) - ax.legend((rects[0], rect2[0]), ('Test', 'Train')) - ax.set_ylim(-0.1, 1.1) - ax.set_xticks(np.arange(nbResults) + barWidth) - ax.set_xticklabels(names, rotation="vertical") - f.tight_layout() - f.savefig(currentDirectory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + dataBaseName + "-Mean_on_" - + str(statsIter) + "_iter-" + metricName + ".png") - plt.close() + trainMeans, trainSTDs = numpy_mean_and_std(scores["trainScores"]) + testMeans, testSTDs = numpy_mean_and_std(scores["testScores"]) + + names = np.array([name for name in classifiersDict.keys()]) + fileName = currentDirectory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + dataBaseName + "-Mean_on_" + str(statsIter) + "_iter-" + metricName + ".png" + nbResults = names.shape[0] + + plotMetricScores(trainScores=trainMeans, testScores=testMeans, names=names, nbResults=nbResults, + metricName=metricName, fileName=fileName, tag=" averaged", + train_STDs=trainSTDs, test_STDs=testSTDs) + def iterCmap(statsIter): @@ -510,40 +587,18 @@ def publishIterBiclassExampleErrors(iterResults, directory, labelsDictionary, cl def publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames, dataBaseName, directory, statsIter, minSize=10): for metricName, scores in iterMulticlassResults["metricsScores"].items(): - trainScores = scores["trainScores"] - testScores = scores["testScores"] - trainMeans = np.mean(trainScores, axis=1) - testMeans = np.mean(testScores, axis=1) - trainSTDs = np.std(trainScores, axis=1) - testSTDs = np.std(testScores, axis=1) - nbResults = len(trainMeans) - names = classifiersNames - size=nbResults - if nbResults<minSize: - size=minSize - figKW = {"figsize" : (size, size/3)} - f, ax = plt.subplots(nrows=1, ncols=1, **figKW) - barWidth = 0.35 # the width of the bars - sorted_indices = np.argsort(testMeans) - testMeans = testMeans[sorted_indices] - testSTDs = testSTDs[sorted_indices] - trainSTDs = trainSTDs[sorted_indices] - trainMeans = trainMeans[sorted_indices] - names = np.array(names)[sorted_indices] - - ax.set_title(metricName + " for each classifier") - rects = ax.bar(range(nbResults), testMeans, barWidth, color="r", yerr=testSTDs) - rect2 = ax.bar(np.arange(nbResults) + barWidth, trainMeans, barWidth, color="0.7", yerr=trainSTDs) - autolabel(rects, ax, set=1) - autolabel(rect2, ax, set=2) - ax.set_ylim(-0.1, 1.1) - ax.legend((rects[0], rect2[0]), ('Test', 'Train')) - ax.set_xticks(np.arange(nbResults) + barWidth) - ax.set_xticklabels(names, rotation="vertical") - f.tight_layout() - f.savefig(directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + dataBaseName + "-Mean_on_" - + str(statsIter) + "_iter-" + metricName + ".png") - plt.close() + + trainMeans, trainSTDs = numpy_mean_and_std(scores["trainScores"]) + testMeans, testSTDs = numpy_mean_and_std(scores["testScores"]) + + nbResults = classifiersNames.shape[0] + + fileName = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-" + dataBaseName + "-Mean_on_" + str(statsIter) + "_iter-" + metricName + ".png" + + plotMetricScores(trainScores=trainMeans, testScores=testMeans, names=classifiersNames, nbResults=nbResults, + metricName=metricName, fileName=fileName, tag=" averaged multiclass", + train_STDs=trainSTDs, test_STDs=testSTDs) + def publishIterMulticlassExampleErrors(iterMulticlassResults, directory, classifiersNames, statsIter, minSize=10): @@ -641,6 +696,7 @@ def analyzeIterMulticlass(multiclassResults, directory, statsIter, metrics, data iterMulticlassResults["errorOnExamples"][classifierIndex, :] += classifierResults["errorOnExample"] logging.debug("Start:\t Getting mean results for multiclass classification") + classifiersNames = np.array(classifiersNames) publishIterMulticlassMetricsScores(iterMulticlassResults, classifiersNames, dataBaseName, directory, statsIter) publishIterMulticlassExampleErrors(iterMulticlassResults, directory, classifiersNames, statsIter)