diff --git a/config_files/config_test.yml b/config_files/config_test.yml
index 5a4d59df8fc4edc1066c69fb6d7d99b427f66f15..7f1aa89e1133b4079de02901147d4966a0c1d6d5 100644
--- a/config_files/config_test.yml
+++ b/config_files/config_test.yml
@@ -1,7 +1,7 @@
# The base configuration of the benchmark
Base :
log: true
- name: ["Plausible"]
+ name: ["plausible"]
label: "_"
type: ".hdf5"
views:
@@ -22,9 +22,9 @@ Classification:
nb_folds: 2
nb_class: 2
classes:
- type: ["multiview"]
- algos_monoview: ["all"]
- algos_multiview: ["mumbo"]
+ type: ["monoview",]
+ algos_monoview: ["adaboost", "decision_tree"]
+ algos_multiview: ["all"]
stats_iter: 2
metrics: ["accuracy_score", "f1_score"]
metric_princ: "f1_score"
diff --git a/multiview_platform/mono_multi_view_classifiers/exec_classif.py b/multiview_platform/mono_multi_view_classifiers/exec_classif.py
index ce66f7057c9867ecee20a5c441b2049bc8f18c81..88ae2c8717d873d46a2ceb85f1144abcc0adb5c5 100644
--- a/multiview_platform/mono_multi_view_classifiers/exec_classif.py
+++ b/multiview_platform/mono_multi_view_classifiers/exec_classif.py
@@ -581,8 +581,7 @@ def exec_one_benchmark_multicore(nb_cores=-1, labels_dictionary=None,
benchmark=None, views=None, views_indices=None,
flag=None, labels=None,
exec_monoview_multicore=exec_monoview_multicore,
- exec_multiview_multicore=exec_multiview_multicore,
- init_multiview_arguments=init_multiview_arguments):
+ exec_multiview_multicore=exec_multiview_multicore,):
"""Used to run a benchmark using multiple cores. ExecMonoview_multicore, initMultiviewArguments and
exec_multiview_multicore args are only used for tests"""
@@ -648,13 +647,11 @@ def exec_one_benchmark_mono_core(dataset_var=None, labels_dictionary=None,
hyper_param_search=None, metrics=None,
argument_dictionaries=None,
benchmark=None, views=None, views_indices=None,
- flag=None, labels=None,
- exec_monoview_multicore=exec_monoview_multicore,
- exec_multiview_multicore=exec_multiview_multicore,
- init_multiview_arguments=init_multiview_arguments):
+ flag=None, labels=None,):
results_monoview, labels_names = benchmark_init(directory,
classification_indices, labels,
labels_dictionary, k_folds)
+ logging.getLogger('matplotlib.font_manager').disabled = True
logging.debug("Start:\t monoview benchmark")
for arguments in argument_dictionaries["monoview"]:
X = dataset_var.get_v(arguments["view_index"])
@@ -759,7 +756,6 @@ def exec_benchmark(nb_cores, stats_iter, nb_multiclass,
benchmark_arguments_dictionaries[0])]
else:
for arguments in benchmark_arguments_dictionaries:
- print(arguments)
results += [exec_one_benchmark_mono_core(dataset_var=dataset_var, **arguments)]
logging.debug("Done:\t Executing all the needed biclass benchmarks")
@@ -777,7 +773,8 @@ def exec_benchmark(nb_cores, stats_iter, nb_multiclass,
directory,
labels_dictionary,
nb_examples,
- nb_labels)
+ nb_labels,
+ dataset_var.example_ids)
logging.debug("Done:\t Analyzing predictions")
delete(benchmark_arguments_dictionaries, nb_cores, dataset_var)
return results_mean_stds
diff --git a/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py b/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
index 0b7b8f5bd4d7dcd1107149b821fb6456ff7b236b..875e3763ab77ea4ea771e6653aa80e2242b87ef2 100644
--- a/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
+++ b/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
@@ -97,15 +97,19 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
logging.debug("Start:\t Predicting")
y_train_pred = classifier.predict(X_train)
y_test_pred = classifier.predict(X_test)
- full_labels_pred = np.zeros(Y.shape, dtype=int) - 100
+
+ #Filling the full prediction in the right order
+ full_pred = np.zeros(Y.shape, dtype=int) - 100
for trainIndex, index in enumerate(classificationIndices[0]):
- full_labels_pred[index] = y_train_pred[trainIndex]
+ full_pred[index] = y_train_pred[trainIndex]
for testIndex, index in enumerate(classificationIndices[1]):
- full_labels_pred[index] = y_test_pred[testIndex]
+ full_pred[index] = y_test_pred[testIndex]
+
if X_test_multiclass != []:
y_test_multiclass_pred = classifier.predict(X_test_multiclass)
else:
y_test_multiclass_pred = []
+
logging.debug("Done:\t Predicting")
t_end = time.time() - t_start
@@ -124,7 +128,7 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
logging.debug("Done:\t Getting results")
logging.debug("Start:\t Saving preds")
- saveResults(stringAnalysis, outputFileName, full_labels_pred, y_train_pred,
+ saveResults(stringAnalysis, outputFileName, full_pred, y_train_pred,
y_train, imagesAnalysis, y_test)
logging.info("Done:\t Saving results")
@@ -132,7 +136,7 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
if testFoldsPreds is None:
testFoldsPreds = y_train_pred
return monoview_utils.MonoviewResult(viewIndex, classifier_name, feat, metricsScores,
- full_labels_pred, clKWARGS,
+ full_pred, clKWARGS,
y_test_multiclass_pred, testFoldsPreds)
# return viewIndex, [CL_type, feat, metricsScores, full_labels_pred, clKWARGS, y_test_multiclass_pred, testFoldsPreds]
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview/analyze_results.py b/multiview_platform/mono_multi_view_classifiers/multiview/analyze_results.py
index 90637f5d70b256275e0cad083701c3f748b2a422..aa305849e6903b42bf63eb9e7b440ec3a20f85c6 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview/analyze_results.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview/analyze_results.py
@@ -66,14 +66,8 @@ def getTotalMetricScores(metric, trainLabels, testLabels, validationIndices,
enumerate(metric[1]))
else:
metricKWARGS = {}
- try:
- trainScore = metricModule.score(labels[learningIndices], trainLabels,
+ trainScore = metricModule.score(labels[learningIndices], trainLabels,
**metricKWARGS)
- except:
- print(labels[learningIndices])
- print(trainLabels)
- import pdb;
- pdb.set_trace()
testScore = metricModule.score(labels[validationIndices], testLabels,
**metricKWARGS)
return [trainScore, testScore]
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py b/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
index 8006f46e71ba3c90d4f5626d765045750cfe13bf..4c5e34719f0692260492bd4b1b95524a1d756bb5 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
@@ -16,11 +16,14 @@ class MultiviewResult(object):
self.y_test_multiclass_pred = test_labels_multiclass
def get_classifier_name(self):
- multiview_classifier_module = getattr(multiview_classifiers,
- self.classifier_name)
- multiview_classifier = getattr(multiview_classifier_module,
- multiview_classifier_module.classifier_class_name)(42)
- return multiview_classifier.short_name
+ try:
+ multiview_classifier_module = getattr(multiview_classifiers,
+ self.classifier_name)
+ multiview_classifier = getattr(multiview_classifier_module,
+ multiview_classifier_module.classifier_class_name)(42)
+ return multiview_classifier.short_name
+ except:
+ return self.classifier_name
def get_names(classed_list):
diff --git a/multiview_platform/mono_multi_view_classifiers/result_analysis.py b/multiview_platform/mono_multi_view_classifiers/result_analysis.py
index 661b84afcf664b0b57869e35ffc66bf374a14385..41e9abcda7057ffcf1c3c7b877ca73d3f05ef0d8 100644
--- a/multiview_platform/mono_multi_view_classifiers/result_analysis.py
+++ b/multiview_platform/mono_multi_view_classifiers/result_analysis.py
@@ -61,6 +61,193 @@ def plot_results_noise(directory, noise_results, metric_to_plot, name, width=0.1
df.to_csv(directory+name+"_noise_analysis.csv")
+def plot_metric_scores(train_scores, test_scores, names, nb_results, metric_name,
+ file_name,
+ tag="", train_STDs=None, test_STDs=None):
+ r"""Used to plot and save the score barplot for a specific metric.
+
+ Parameters
+ ----------
+ train_scores : list or np.array of floats
+ The scores of each classifier on the training set.
+ test_scores : list or np.array of floats
+ The scores of each classifier on the testing set.
+ names : list or np.array of strs
+ The names of all the classifiers.
+ nb_results: int
+ The number of classifiers to plot.
+ metric_name : str
+ The plotted metric's name
+ file_name : str
+ The name of the file where the figure will be saved.
+ tag : str
+ Some text to personalize the title, must start with a whitespace.
+ train_STDs : np.array of floats or None
+ The array containing the standard deviations for the averaged scores on the training set.
+ test_STDs : np.array of floats or None
+ The array containing the standard deviations for the averaged scores on the testing set.
+
+ Returns
+ -------
+ """
+
+ figKW, barWidth = get_fig_size(nb_results)
+
+ names, train_scores, test_scores, train_STDs, test_STDs = sort_by_test_score(
+ train_scores, test_scores, names,
+ train_STDs, test_STDs)
+
+ f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
+ ax.set_title(metric_name + "\n" + tag + " scores for each classifier")
+
+ rects = ax.bar(range(nb_results), test_scores, barWidth, color="0.1",
+ yerr=test_STDs)
+ rect2 = ax.bar(np.arange(nb_results) + barWidth, train_scores, barWidth,
+ color="0.8", yerr=train_STDs)
+ autolabel(rects, ax, set=1, std=test_STDs)
+ autolabel(rect2, ax, set=2, std=train_STDs)
+ ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
+ ax.set_ylim(-0.1, 1.1)
+ ax.set_xticks(np.arange(nb_results) + barWidth)
+ ax.set_xticklabels(names, rotation="vertical")
+
+ try:
+ plt.tight_layout()
+ except:
+ pass
+ f.savefig(file_name + '.png', transparent=True)
+ plt.close()
+ import pandas as pd
+ if train_STDs is None:
+ dataframe = pd.DataFrame(np.transpose(np.concatenate((
+ train_scores.reshape((train_scores.shape[0], 1)),
+ test_scores.reshape((train_scores.shape[0], 1))), axis=1)),
+ columns=names)
+ else:
+ dataframe = pd.DataFrame(np.transpose(np.concatenate((
+ train_scores.reshape((train_scores.shape[0], 1)),
+ train_STDs.reshape((train_scores.shape[0], 1)),
+ test_scores.reshape((train_scores.shape[0], 1)),
+ test_STDs.reshape((train_scores.shape[0], 1))), axis=1)),
+ columns=names)
+ dataframe.to_csv(file_name + ".csv")
+
+
+def plot_2d(data, classifiers_names, nbClassifiers, nbExamples,
+ fileName, minSize=10,
+ width_denominator=2.0, height_denominator=20.0, stats_iter=1,
+ use_plotly=True, example_ids=None):
+ r"""Used to generate a 2D plot of the errors.
+
+ Parameters
+ ----------
+ data : np.array of shape `(nbClassifiers, nbExamples)`
+ A matrix with zeros where the classifier failed to classifiy the example, ones where it classified it well
+ and -100 if the example was not classified.
+ classifiers_names : list of str
+ The names of the classifiers.
+ nbClassifiers : int
+ The number of classifiers.
+ nbExamples : int
+ The number of examples.
+ nbCopies : int
+ The number of times the data is copied (classifier wise) in order for the figure to be more readable
+ fileName : str
+ The name of the file in which the figure will be saved ("error_analysis_2D.png" will be added at the end)
+ minSize : int, optinal, default: 10
+ The minimum width and height of the figure.
+ width_denominator : float, optional, default: 1.0
+ To obtain the image width, the number of classifiers will be divided by this number.
+ height_denominator : float, optional, default: 1.0
+ To obtain the image width, the number of examples will be divided by this number.
+ stats_iter : int, optional, default: 1
+ The number of statistical iterations realized.
+
+ Returns
+ -------
+ """
+ fig, ax = plt.subplots(nrows=1, ncols=1,)
+ cmap, norm = iterCmap(stats_iter)
+ cax = plt.imshow(data, cmap=cmap, norm=norm,
+ aspect='auto')
+ plt.title('Errors depending on the classifier')
+ ticks = np.arange(0, nbClassifiers, 1)
+ labels = classifiers_names
+ plt.xticks(ticks, labels, rotation="vertical")
+ cbar = fig.colorbar(cax, ticks=[-100 * stats_iter / 2, 0, stats_iter])
+ cbar.ax.set_yticklabels(['Unseen', 'Always Wrong', 'Always Right'])
+
+ fig.savefig(fileName + "error_analysis_2D.png", bbox_inches="tight", transparent=True)
+ plt.close()
+ ### The following part is used to generate an interactive graph.
+ if use_plotly:
+ import plotly
+ fig = plotly.graph_objs.Figure(data=plotly.graph_objs.Heatmap(
+ x=list(classifiers_names),
+ y=example_ids,
+ z=data,
+ colorscale="Greys",
+ reversescale=True))
+ fig.update_layout(
+ xaxis={"showgrid": False, "showticklabels": False, "ticks": ''},
+ yaxis={"showgrid": False, "showticklabels": False, "ticks": ''})
+ plotly.offline.plot(fig, filename=fileName + "error_analysis_2D.html", auto_open=False)
+ del fig
+
+
+def plot_errors_bar(error_on_examples, nbClassifiers, nbExamples, fileName):
+ r"""Used to generate a barplot of the muber of classifiers that failed to classify each examples
+
+ Parameters
+ ----------
+ error_on_examples : np.array of shape `(nbExamples,)`
+ An array counting how many classifiers failed to classifiy each examples.
+ classifiers_names : list of str
+ The names of the classifiers.
+ nbClassifiers : int
+ The number of classifiers.
+ nbExamples : int
+ The number of examples.
+ fileName : str
+ The name of the file in which the figure will be saved ("error_analysis_2D.png" will be added at the end)
+
+ Returns
+ -------
+ """
+ fig, ax = plt.subplots()
+ x = np.arange(nbExamples)
+ plt.bar(x, error_on_examples)
+ plt.ylim([0, nbClassifiers])
+ plt.title("Number of classifiers that failed to classify each example")
+ fig.savefig(fileName + "error_analysis_bar.png", transparent=True)
+ plt.close()
+
+
+def iterCmap(statsIter):
+ r"""Used to generate a colormap that will have a tick for each iteration : the whiter the better.
+
+ Parameters
+ ----------
+ statsIter : int
+ The number of statistical iterations.
+
+ Returns
+ -------
+ cmap : matplotlib.colors.ListedColorMap object
+ The colormap.
+ norm : matplotlib.colors.BoundaryNorm object
+ The bounds for the colormap.
+ """
+ cmapList = ["red", "0.0"] + [str(float((i + 1)) / statsIter) for i in
+ range(statsIter)]
+ cmap = mpl.colors.ListedColormap(cmapList)
+ bounds = [-100 * statsIter - 0.5, -0.5]
+ for i in range(statsIter):
+ bounds.append(i + 0.5)
+ bounds.append(statsIter + 0.5)
+ norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
+ return cmap, norm
+
def autolabel(rects, ax, set=1, std=None):
r"""Used to print the score below the bars.
@@ -97,6 +284,34 @@ def autolabel(rects, ax, set=1, std=None):
"%.2f" % height, weight=weight,
ha='center', va='bottom', size="small")
+def get_fig_size(nb_results, min_size=15, multiplier=1.0, bar_width=0.35):
+ r"""Used to get the image size to save the figure and the bar width, depending on the number of scores to plot.
+
+ Parameters
+ ----------
+ nb_results : int
+ The number of couple of bar to plot.
+ min_size : int
+ The minimum size of the image, if there are few classifiers to plot.
+ multiplier : float
+ The ratio between the image size and the number of classifiers.
+ bar_width : float
+ The width of the bars in the figure. Mainly here to centralize bar_width.
+
+ Returns
+ -------
+ fig_kwargs : dict of arguments
+ The argument restraining the size of the figure, usable directly in the `subplots` function of
+ `matplotlib.pyplot`.
+ bar_width : float
+ The width of the bars in the figure. Mainly here to centralize bar_width.
+ """
+ size = nb_results * multiplier
+ if size < min_size:
+ size = min_size
+ fig_kwargs = {"figsize": (size, size / 3)}
+ return fig_kwargs, bar_width
+
def get_metrics_scores_biclass(metrics, results):
r"""Used to extract metrics scores in case of biclass classification
@@ -106,7 +321,7 @@ def get_metrics_scores_biclass(metrics, results):
metrics : list of lists
The metrics names with configuration metrics[i][0] = name of metric i
results : list of MonoviewResult and MultiviewResults objects
- A list containing all the resluts for all the monoview experimentations.
+ A list containing all the results for all the monoview experimentations.
Returns
-------
@@ -117,25 +332,28 @@ def get_metrics_scores_biclass(metrics, results):
-`metricScores[metric_name]["train_scores"]` is a list of all the available classifiers scores on the train set,
-`metricScores[metric_name]["test_scores"]` is a list of all the available classifiers scores on the test set.
"""
- metrics_scores = {}
+ classifier_names=[]
+ classifier_names = [classifierResult.get_classifier_name()
+ for classifierResult in results
+ if classifierResult.get_classifier_name()
+ not in classifier_names ]
+ metrics_scores = dict((metric[0], pd.DataFrame(data=np.zeros((2,
+ len(classifier_names))),
+ index=["train", "test"],
+ columns=classifier_names))
+ for metric in metrics)
for metric in metrics:
- classifiers_names = []
- train_scores = []
- test_scores = []
-
for classifierResult in results:
- train_scores.append(classifierResult.metrics_scores[metric[0]][0])
- test_scores.append(classifierResult.metrics_scores[metric[0]][1])
- classifiers_names.append(classifierResult.get_classifier_name())
+ metrics_scores[metric[0]].loc["train", classifierResult.get_classifier_name()] = classifierResult.metrics_scores[metric[0]][0]
+ metrics_scores[metric[0]].loc[
+ "test", classifierResult.get_classifier_name()] = \
+ classifierResult.metrics_scores[metric[0]][1]
- metrics_scores[metric[0]] = {"classifiers_names": classifiers_names,
- "train_scores": train_scores,
- "test_scores": test_scores}
return metrics_scores
-def getExampleErrorsBiclass(groud_truth, results):
+def get_example_errors_biclass(groud_truth, results):
r"""Used to get for each classifier and each example whether the classifier has misclassified the example or not.
Parameters
@@ -154,46 +372,15 @@ def getExampleErrorsBiclass(groud_truth, results):
"""
example_errors = {}
- for classifierResult in results:
- error_on_examples = np.equal(classifierResult.full_labels_pred,
+ for classifier_result in results:
+ error_on_examples = np.equal(classifier_result.full_labels_pred,
groud_truth).astype(int)
- unseenExamples = np.where(groud_truth == -100)[0]
- error_on_examples[unseenExamples] = -100
- example_errors[classifierResult.get_classifier_name()] = {
- "error_on_examples": error_on_examples}
-
+ unseen_examples = np.where(groud_truth == -100)[0]
+ error_on_examples[unseen_examples] = -100
+ example_errors[classifier_result.get_classifier_name()] = error_on_examples
return example_errors
-def get_fig_size(nb_results, min_size=15, multiplier=1.0, bar_width=0.35):
- r"""Used to get the image size to save the figure and the bar width, depending on the number of scores to plot.
-
- Parameters
- ----------
- nb_results : int
- The number of couple of bar to plot.
- min_size : int
- The minimum size of the image, if there are few classifiers to plot.
- multiplier : float
- The ratio between the image size and the number of classifiers.
- bar_width : float
- The width of the bars in the figure. Mainly here to centralize bar_width.
-
- Returns
- -------
- fig_kwargs : dict of arguments
- The argument restraining the size of the figure, usable directly in the `subplots` function of
- `matplotlib.pyplot`.
- bar_width : float
- The width of the bars in the figure. Mainly here to centralize bar_width.
- """
- size = nb_results * multiplier
- if size < min_size:
- size = min_size
- fig_kwargs = {"figsize": (size, size / 3)}
- return fig_kwargs, bar_width
-
-
def sort_by_test_score(train_scores, test_scores, names, train_STDs=None,
test_STDs=None):
r"""Used to sort the results (names and both scores) in descending test score order.
@@ -239,77 +426,7 @@ def sort_by_test_score(train_scores, test_scores, names, train_STDs=None,
return sorted_names, sorted_train_scores, sorted_test_scores, sorted_train_STDs, sorted_test_STDs
-def plotMetricScores(train_scores, test_scores, names, nb_results, metric_name,
- file_name,
- tag="", train_STDs=None, test_STDs=None):
- r"""Used to plot and save the score barplot for a specific metric.
-
- Parameters
- ----------
- train_scores : list or np.array of floats
- The scores of each classifier on the training set.
- test_scores : list or np.array of floats
- The scores of each classifier on the testing set.
- names : list or np.array of strs
- The names of all the classifiers.
- nb_results: int
- The number of classifiers to plot.
- metric_name : str
- The plotted metric's name
- file_name : str
- The name of the file where the figure will be saved.
- tag : str
- Some text to personalize the title, must start with a whitespace.
- train_STDs : np.array of floats or None
- The array containing the standard deviations for the averaged scores on the training set.
- test_STDs : np.array of floats or None
- The array containing the standard deviations for the averaged scores on the testing set.
-
- Returns
- -------
- """
-
- figKW, barWidth = get_fig_size(nb_results)
- names, train_scores, test_scores, train_STDs, test_STDs = sort_by_test_score(
- train_scores, test_scores, names,
- train_STDs, test_STDs)
-
- f, ax = plt.subplots(nrows=1, ncols=1, **figKW)
- ax.set_title(metric_name + "\n" + tag + " scores for each classifier")
-
- rects = ax.bar(range(nb_results), test_scores, barWidth, color="0.1",
- yerr=test_STDs)
- rect2 = ax.bar(np.arange(nb_results) + barWidth, train_scores, barWidth,
- color="0.8", yerr=train_STDs)
- autolabel(rects, ax, set=1, std=test_STDs)
- autolabel(rect2, ax, set=2, std=train_STDs)
- print("nb_results", nb_results)
- ax.legend((rects[0], rect2[0]), ('Test', 'Train'))
- ax.set_ylim(-0.1, 1.1)
- ax.set_xticks(np.arange(nb_results) + barWidth)
- ax.set_xticklabels(names, rotation="vertical")
-
- try:
- plt.tight_layout()
- except:
- pass
- f.savefig(file_name + '.png', transparent=True)
- plt.close()
- import pandas as pd
- if train_STDs is None:
- dataframe = pd.DataFrame(np.transpose(np.concatenate((
- train_scores.reshape((train_scores.shape[0], 1)),
- test_scores.reshape((train_scores.shape[0], 1))), axis=1)),
- columns=names)
- else:
- dataframe = pd.DataFrame(np.transpose(np.concatenate((
- train_scores.reshape((train_scores.shape[0], 1)),
- train_STDs.reshape((train_scores.shape[0], 1)),
- test_scores.reshape((train_scores.shape[0], 1)),
- test_STDs.reshape((train_scores.shape[0], 1))), axis=1)),
- columns=names)
- dataframe.to_csv(file_name + ".csv")
def publishMetricsGraphs(metrics_scores, directory, database_name, labels_names):
@@ -332,151 +449,40 @@ def publishMetricsGraphs(metrics_scores, directory, database_name, labels_names)
results
"""
results=[]
- for metric_name, metric_scores in metrics_scores.items():
+ for metric_name, metric_dataframe in metrics_scores.items():
logging.debug(
"Start:\t Biclass score graph generation for " + metric_name)
-
- nb_results = len(metric_scores["test_scores"])
- file_name = directory + time.strftime(
- "%Y_%m_%d-%H_%M_%S") + "-" + database_name + "-" + "_vs_".join(
- labels_names) + "-" + metric_name
-
- plotMetricScores(np.array(metric_scores["train_scores"]),
- np.array(metric_scores["test_scores"]),
- np.array(metric_scores["classifiers_names"]), nb_results,
- metric_name, file_name,
- tag=" " + " vs ".join(labels_names))
-
- logging.debug(
- "Done:\t Biclass score graph generation for " + metric_name)
- results+=[[classifiers_name, metric_name, testMean, testSTD]
- for classifiers_name, testMean, testSTD in zip(np.array(metric_scores["classifiers_names"]),
- np.array(metric_scores["test_scores"]),
- np.zeros(len(np.array(metric_scores["test_scores"]))))]
+ train_scores, test_scores, classifier_names, \
+ file_name, nb_results,results = init_plot(results, metric_name,
+ metric_dataframe, directory,
+ database_name, labels_names)
+
+ plot_metric_scores(train_scores, test_scores, classifier_names,
+ nb_results, metric_name, file_name,
+ tag=" "+" vs ".join(labels_names))
+ logging.debug("Done:\t Biclass score graph generation for "+metric_name)
return results
-def iterCmap(statsIter):
- r"""Used to generate a colormap that will have a tick for each iteration : the whiter the better.
-
- Parameters
- ----------
- statsIter : int
- The number of statistical iterations.
-
- Returns
- -------
- cmap : matplotlib.colors.ListedColorMap object
- The colormap.
- norm : matplotlib.colors.BoundaryNorm object
- The bounds for the colormap.
- """
- cmapList = ["red", "0.0"] + [str(float((i + 1)) / statsIter) for i in
- range(statsIter)]
- cmap = mpl.colors.ListedColormap(cmapList)
- bounds = [-100 * statsIter - 0.5, -0.5]
- for i in range(statsIter):
- bounds.append(i + 0.5)
- bounds.append(statsIter + 0.5)
- norm = mpl.colors.BoundaryNorm(bounds, cmap.N)
- return cmap, norm
-
-
-def publish2Dplot(data, classifiers_names, nbClassifiers, nbExamples, nbCopies,
- fileName, minSize=10,
- width_denominator=2.0, height_denominator=20.0, stats_iter=1,
- use_plotly=False, example_ids=None):
- r"""Used to generate a 2D plot of the errors.
-
- Parameters
- ----------
- data : np.array of shape `(nbClassifiers, nbExamples)`
- A matrix with zeros where the classifier failed to classifiy the example, ones where it classified it well
- and -100 if the example was not classified.
- classifiers_names : list of str
- The names of the classifiers.
- nbClassifiers : int
- The number of classifiers.
- nbExamples : int
- The number of examples.
- nbCopies : int
- The number of times the data is copied (classifier wise) in order for the figure to be more readable
- fileName : str
- The name of the file in which the figure will be saved ("error_analysis_2D.png" will be added at the end)
- minSize : int, optinal, default: 10
- The minimum width and height of the figure.
- width_denominator : float, optional, default: 1.0
- To obtain the image width, the number of classifiers will be divided by this number.
- height_denominator : float, optional, default: 1.0
- To obtain the image width, the number of examples will be divided by this number.
- stats_iter : int, optional, default: 1
- The number of statistical iterations realized.
-
- Returns
- -------
- """
- figWidth = max(nbClassifiers / width_denominator, minSize)
- figHeight = max(nbExamples / height_denominator, minSize)
- print(figHeight, figWidth, nbClassifiers, nbExamples)
- figKW = {"figsize": (figWidth, figHeight)}
- fig, ax = plt.subplots(nrows=1, ncols=1,)# **figKW)
- cmap, norm = iterCmap(stats_iter)
- cax = plt.imshow(data, cmap=cmap, norm=norm,
- aspect='auto')
- plt.title('Errors depending on the classifier')
- ticks = np.arange(nbCopies / 2 - 0.5, nbClassifiers * nbCopies, nbCopies)
- labels = classifiers_names
- plt.xticks(ticks, labels, rotation="vertical")
- cbar = fig.colorbar(cax, ticks=[-100 * stats_iter / 2, 0, stats_iter])
- cbar.ax.set_yticklabels(['Unseen', 'Always Wrong', 'Always Right'])
- # fig.tight_layout()
-
- fig.savefig(fileName + "error_analysis_2D.png", bbox_inches="tight", transparent=True)
- plt.close()
- ### The following part is used to generate an interactive graph.
- if use_plotly:
- import plotly
- fig = plotly.graph_objs.go.Figure(data=plotly.graph_objs.go.Heatmap(
- x=classifiers_names,
- y=example_ids,
- z=data,
- colorscale="Greys",
- ))
- fig.update_layout(
- xaxis={"showgrid": False, "showticklabels": False, "ticks": ''},
- yaxis={"showgrid": False, "showticklabels": False, "ticks": ''})
- plotly.offline.plot(fig, filename='essai.html', auto_open=False)
- del fig
+def init_plot(results, metric_name, metric_dataframe,
+ directory, database_name, labels_names):
-def publishErrorsBarPlot(error_on_examples, nbClassifiers, nbExamples, fileName):
- r"""Used to generate a barplot of the muber of classifiers that failed to classify each examples
+ train = np.array(metric_dataframe.loc["train"])
+ test = np.array(metric_dataframe.loc["test"])
+ classifier_names = np.array(metric_dataframe.columns)
- Parameters
- ----------
- error_on_examples : np.array of shape `(nbExamples,)`
- An array counting how many classifiers failed to classifiy each examples.
- classifiers_names : list of str
- The names of the classifiers.
- nbClassifiers : int
- The number of classifiers.
- nbExamples : int
- The number of examples.
- fileName : str
- The name of the file in which the figure will be saved ("error_analysis_2D.png" will be added at the end)
+ nb_results = metric_dataframe.shape[1]
- Returns
- -------
- """
- fig, ax = plt.subplots()
- x = np.arange(nbExamples)
- plt.bar(x, error_on_examples)
- plt.ylim([0, nbClassifiers])
- plt.title("Number of classifiers that failed to classify each example")
- fig.savefig(fileName + "error_analysis_bar.png", transparent=True)
- plt.close()
+ file_name = directory + time.strftime(
+ "%Y_%m_%d-%H_%M_%S") + "-" + database_name + "-" + "_vs_".join(
+ labels_names) + "-" + metric_name
+ results += [[classifiers_name, metric_name, testMean, testSTD]
+ for classifiers_name, testMean, testSTD in
+ zip(classifier_names, test, np.zeros(len(test)))]
+ return train, test, classifier_names, file_name, nb_results, results
-def gen_error_data(example_errors, base_file_name, nbCopies=2):
+def gen_error_data(example_errors):
r"""Used to format the error data in order to plot it efficiently. The data is saves in a `.csv` file.
Parameters
@@ -510,42 +516,38 @@ def gen_error_data(example_errors, base_file_name, nbCopies=2):
error_on_examples : np.array of shape `(nbExamples,)`
An array counting how many classifiers failed to classifiy each examples.
"""
- nbClassifiers = len(example_errors)
- nbExamples = len(list(example_errors.values())[0]["error_on_examples"])
- classifiers_names = example_errors.keys()
+ nb_classifiers = len(example_errors)
+ nb_examples = len(list(example_errors.values())[0])
+ classifiers_names = list(example_errors.keys())
- data = np.zeros((nbExamples, nbClassifiers * nbCopies))
- temp_data = np.zeros((nbExamples, nbClassifiers))
+ data_2d = np.zeros((nb_examples, nb_classifiers))
for classifierIndex, (classifier_name, error_on_examples) in enumerate(
example_errors.items()):
- for iter_index in range(nbCopies):
- data[:, classifierIndex * nbCopies + iter_index] = error_on_examples[
- "error_on_examples"]
- temp_data[:, classifierIndex] = error_on_examples["error_on_examples"]
- error_on_examples = -1 * np.sum(data, axis=1) / nbCopies + nbClassifiers
-
- np.savetxt(base_file_name + "2D_plot_data.csv", data, delimiter=",")
- np.savetxt(base_file_name + "bar_plot_data.csv", temp_data, delimiter=",")
+ data_2d[:, classifierIndex] = error_on_examples
+ error_on_examples = -1 * np.sum(data_2d, axis=1) / nb_classifiers
- return nbClassifiers, nbExamples, nbCopies, classifiers_names, data, error_on_examples
+ return nb_classifiers, nb_examples, classifiers_names, data_2d, error_on_examples
-def publishExampleErrors(example_errors, directory, databaseName, labels_names):
+def publishExampleErrors(example_errors, directory, databaseName, labels_names, example_ids):
logging.debug("Start:\t Biclass Label analysis figure generation")
base_file_name = directory + time.strftime(
"%Y_%m_%d-%H_%M_%S") + "-" + databaseName + "-" + "_vs_".join(
labels_names) + "-"
- nbClassifiers, nbExamples, nCopies, classifiers_names, data, error_on_examples = gen_error_data(
- example_errors,
- base_file_name)
+ nb_classifiers, nb_examples, classifiers_names, \
+ data_2d, error_on_examples = gen_error_data(example_errors)
- publish2Dplot(data, classifiers_names, nbClassifiers, nbExamples, nCopies,
- base_file_name)
+ np.savetxt(base_file_name + "2D_plot_data.csv", data_2d, delimiter=",")
+ np.savetxt(base_file_name + "bar_plot_data.csv", error_on_examples,
+ delimiter=",")
+
+ plot_2d(data_2d, classifiers_names, nb_classifiers, nb_examples,
+ base_file_name, example_ids=example_ids)
- publishErrorsBarPlot(error_on_examples, nbClassifiers, nbExamples,
- base_file_name)
+ plot_errors_bar(error_on_examples, nb_classifiers, nb_examples,
+ base_file_name)
logging.debug("Done:\t Biclass Label analysis figures generation")
@@ -571,7 +573,7 @@ def get_arguments(benchmark_argument_dictionaries, flag):
return benchmarkArgumentDictionary
-def analyze_biclass(results, benchmark_argument_dictionaries, stats_iter, metrics):
+def analyze_biclass(results, benchmark_argument_dictionaries, stats_iter, metrics, example_ids):
r"""Used to extract and format the results of the different biclass experimentations performed.
Parameters
@@ -598,7 +600,7 @@ def analyze_biclass(results, benchmark_argument_dictionaries, stats_iter, metric
label combination, regrouping the scores for each metrics and the information useful to plot errors on examples.
"""
logging.debug("Srart:\t Analzing all biclass resuls")
- biclass_results = [{} for _ in range(stats_iter)]
+ biclass_results = {}
for flag, result in results:
iteridex, [classifierPositive, classifierNegative] = flag
@@ -606,7 +608,7 @@ def analyze_biclass(results, benchmark_argument_dictionaries, stats_iter, metric
arguments = get_arguments(benchmark_argument_dictionaries, flag)
metrics_scores = get_metrics_scores_biclass(metrics, result)
- example_errors = getExampleErrorsBiclass(arguments["labels"], result)
+ example_errors = get_example_errors_biclass(arguments["labels"], result)
directory = arguments["directory"]
@@ -617,12 +619,15 @@ def analyze_biclass(results, benchmark_argument_dictionaries, stats_iter, metric
results = publishMetricsGraphs(metrics_scores, directory, database_name,
labels_names)
publishExampleErrors(example_errors, directory, database_name,
- labels_names)
-
- biclass_results[iteridex][
- str(classifierPositive) + str(classifierNegative)] = {
- "metrics_scores": metrics_scores,
- "example_errors": example_errors}
+ labels_names, example_ids)
+ if not str(classifierPositive) + str(classifierNegative) in biclass_results:
+ biclass_results[str(classifierPositive) + str(classifierNegative)] = {}
+ biclass_results[str(classifierPositive) + str(classifierNegative)][
+ "metrics_scores"] = [i for i in range(stats_iter)]
+ biclass_results[str(classifierPositive) + str(classifierNegative)][
+ "example_errors"] = [i for i in range(stats_iter)]
+ biclass_results[str(classifierPositive) + str(classifierNegative)]["metrics_scores"][iteridex] = metrics_scores
+ biclass_results[str(classifierPositive) + str(classifierNegative)]["example_errors"][iteridex] = example_errors
logging.debug("Done:\t Analzing all biclass resuls")
return results, biclass_results
@@ -702,8 +707,8 @@ def publishMulticlassScores(multiclass_results, metrics, stats_iter, direcories,
"%Y_%m_%d-%H_%M_%S") + "-" + databaseName + "-" + metric[
0] + ".png"
- plotMetricScores(train_scores, validationScores, classifiers_names,
- nbResults, metric[0], fileName, tag=" multiclass")
+ plot_metric_scores(train_scores, validationScores, classifiers_names,
+ nbResults, metric[0], fileName, tag=" multiclass")
logging.debug(
"Done:\t Multiclass score graph generation for " + metric[0])
@@ -712,7 +717,7 @@ def publishMulticlassScores(multiclass_results, metrics, stats_iter, direcories,
def publishMulticlassExmapleErrors(multiclass_results, directories,
- databaseName):
+ databaseName, example_ids):
for iter_index, multiclassResult in enumerate(multiclass_results):
directory = directories[iter_index]
logging.debug("Start:\t Multiclass Label analysis figure generation")
@@ -724,18 +729,18 @@ def publishMulticlassExmapleErrors(multiclass_results, directories,
multiclassResult,
base_file_name)
- publish2Dplot(data, classifiers_names, nbClassifiers, nbExamples,
- nCopies, base_file_name)
+ plot_2d(data, classifiers_names, nbClassifiers, nbExamples,
+ nCopies, base_file_name, example_ids=example_ids)
- publishErrorsBarPlot(error_on_examples, nbClassifiers, nbExamples,
- base_file_name)
+ plot_errors_bar(error_on_examples, nbClassifiers, nbExamples,
+ base_file_name)
logging.debug("Done:\t Multiclass Label analysis figure generation")
def analyzeMulticlass(results, stats_iter, benchmark_argument_dictionaries,
nb_examples, nb_labels, multiclass_labels,
- metrics, classification_indices, directories):
+ metrics, classification_indices, directories, example_ids):
"""Used to transform one versus one results in multiclass results and to publish it"""
multiclass_results = [{} for _ in range(stats_iter)]
@@ -787,7 +792,7 @@ def analyzeMulticlass(results, stats_iter, benchmark_argument_dictionaries,
benchmark_argument_dictionaries[0]["args"]["Base"]["name"])
publishMulticlassExmapleErrors(multiclass_results, directories,
benchmark_argument_dictionaries[0][
- "args"].name)
+ "args"].name, example_ids)
return results, multiclass_results
@@ -820,9 +825,9 @@ def publish_iter_biclass_metrics_scores(iter_results, directory, labels_dictiona
stats_iter) + "_iter-" + metricName + ".png"
nbResults = names.shape[0]
- plotMetricScores(trainMeans, testMeans, names, nbResults,
- metricName, fileName, tag=" averaged",
- train_STDs=trainSTDs, test_STDs=testSTDs)
+ plot_metric_scores(trainMeans, testMeans, names, nbResults,
+ metricName, fileName, tag=" averaged",
+ train_STDs=trainSTDs, test_STDs=testSTDs)
results+=[[classifiersName, metricName, testMean, testSTD] for classifiersName, testMean, testSTD in zip(names, testMeans, testSTDs)]
return results
@@ -839,7 +844,7 @@ def gen_error_dat_glob(combi_results, stats_iter, base_file_name):
def publish_iter_biclass_example_errors(iter_results, directory, labels_dictionary,
- classifiers_dict, stats_iter, min_size=10):
+ classifiers_dict, stats_iter, exmaple_ids, min_size=10):
for labelsCombination, combiResults in iter_results.items():
base_file_name = directory + labels_dictionary[
int(labelsCombination[0])] + "-vs-" + \
@@ -847,18 +852,18 @@ def publish_iter_biclass_example_errors(iter_results, directory, labels_dictiona
int(labelsCombination[1])] + "/" + time.strftime(
"%Y_%m_%d-%H_%M_%S") + "-"
classifiers_names = [classifier_name for classifier_name in
- classifiers_dict.values()]
+ classifiers_dict.keys()]
logging.debug(
"Start:\t Global biclass label analysis figure generation")
nbExamples, nbClassifiers, data, error_on_examples = gen_error_dat_glob(
combiResults, stats_iter, base_file_name)
- publish2Dplot(data, classifiers_names, nbClassifiers, nbExamples, 1,
- base_file_name, stats_iter=stats_iter)
+ plot_2d(data, classifiers_names, nbClassifiers, nbExamples, 1,
+ base_file_name, stats_iter=stats_iter, example_ids=exmaple_ids)
- publishErrorsBarPlot(error_on_examples, nbClassifiers * stats_iter,
- nbExamples, base_file_name)
+ plot_errors_bar(error_on_examples, nbClassifiers * stats_iter,
+ nbExamples, base_file_name)
logging.debug(
"Done:\t Global biclass label analysis figures generation")
@@ -878,16 +883,16 @@ def publish_iter_multiclass_metrics_scores(iter_multiclass_results, classifiers_
"%Y_%m_%d-%H_%M_%S") + "-" + data_base_name + "-Mean_on_" + str(
stats_iter) + "_iter-" + metric_name + ".png"
- plotMetricScores(trainMeans, testMeans, classifiers_names, nb_results,
- metric_name, file_name, tag=" averaged multiclass",
- train_STDs=trainSTDs, test_STDs=testSTDs)
+ plot_metric_scores(trainMeans, testMeans, classifiers_names, nb_results,
+ metric_name, file_name, tag=" averaged multiclass",
+ train_STDs=trainSTDs, test_STDs=testSTDs)
results+=[[classifiers_name, metric_name,testMean, testSTD] for classifiers_name, testMean, testSTD in zip(classifiers_names, testMeans, testSTDs)]
return results
def publish_iter_multiclass_example_errors(iter_multiclass_results, directory,
- classifiers_names, stats_iter, min_size=10):
+ classifiers_names, stats_iter, example_ids, min_size=10):
logging.debug(
"Start:\t Global multiclass label analysis figures generation")
base_file_name = directory + time.strftime("%Y_%m_%d-%H_%M_%S") + "-"
@@ -895,11 +900,11 @@ def publish_iter_multiclass_example_errors(iter_multiclass_results, directory,
nb_examples, nb_classifiers, data, error_on_examples = gen_error_dat_glob(
iter_multiclass_results, stats_iter, base_file_name)
- publish2Dplot(data, classifiers_names, nb_classifiers, nb_examples, 1,
- base_file_name, stats_iter=stats_iter)
+ plot_2d(data, classifiers_names, nb_classifiers, nb_examples, 1,
+ base_file_name, stats_iter=stats_iter, example_ids=example_ids)
- publishErrorsBarPlot(error_on_examples, nb_classifiers * stats_iter, nb_examples,
- base_file_name)
+ plot_errors_bar(error_on_examples, nb_classifiers * stats_iter, nb_examples,
+ base_file_name)
logging.debug("Done:\t Global multiclass label analysis figures generation")
@@ -908,8 +913,7 @@ def gen_classifiers_dict(results, metrics):
classifiers_dict = dict((classifier_name, classifierIndex)
for classifierIndex, classifier_name
in enumerate(
- results[0][list(results[0].keys())[0]]["metrics_scores"][metrics[0][0]][
- "classifiers_names"]))
+ list(results[list(results.keys())[0]]["metrics_scores"][0][metrics[0][0]].columns)))
return classifiers_dict, len(classifiers_dict)
@@ -938,12 +942,17 @@ def add_new_metric(iter_biclass_results, metric, labels_combination, nb_classifi
def analyzebiclass_iter(biclass_results, metrics, stats_iter, directory,
- labels_dictionary, data_base_name, nb_examples):
+ labels_dictionary, data_base_name, nb_examples, example_ids):
"""Used to format the results in order to plot the mean results on the iterations"""
iter_biclass_results = {}
classifiers_dict, nb_classifiers = gen_classifiers_dict(biclass_results,
metrics)
+ for label_combination, biclass_result in biclass_results.items():
+ for iter_index, metric_score in enumerate(biclass_result["metrics_scores"]):
+ print(metric_score)
+
+
for iter_index, biclass_result in enumerate(biclass_results):
for labelsComination, results in biclass_result.items():
for metric in metrics:
@@ -978,11 +987,11 @@ def analyzebiclass_iter(biclass_results, metrics, stats_iter, directory,
data_base_name, stats_iter)
publish_iter_biclass_example_errors(iter_biclass_results, directory,
labels_dictionary, classifiers_dict,
- stats_iter)
+ stats_iter, example_ids)
return results
def analyze_iter_multiclass(multiclass_results, directory, stats_iter, metrics,
- data_base_name, nb_examples):
+ data_base_name, nb_examples, example_ids):
"""Used to mean the multiclass results on the iterations executed with different random states"""
logging.debug("Start:\t Getting mean results for multiclass classification")
@@ -1019,19 +1028,19 @@ def analyze_iter_multiclass(multiclass_results, directory, stats_iter, metrics,
iter_multiclass_results, classifiers_names,
data_base_name, directory, stats_iter)
publish_iter_multiclass_example_errors(iter_multiclass_results, directory,
- classifiers_names, stats_iter)
+ classifiers_names, stats_iter, example_ids)
return results
def get_results(results, stats_iter, nb_multiclass, benchmark_argument_dictionaries,
multiclass_labels, metrics,
classification_indices, directories, directory, labels_dictionary,
- nb_examples, nb_labels):
+ nb_examples, nb_labels, example_ids):
"""Used to analyze the results of the previous benchmarks"""
data_base_name = benchmark_argument_dictionaries[0]["args"]["Base"]["name"]
results_means_std, biclass_results = analyze_biclass(results, benchmark_argument_dictionaries,
- stats_iter, metrics)
+ stats_iter, metrics, example_ids)
if nb_multiclass > 1:
results_means_std, multiclass_results = analyzeMulticlass(results, stats_iter,
@@ -1039,12 +1048,12 @@ def get_results(results, stats_iter, nb_multiclass, benchmark_argument_dictionar
nb_examples, nb_labels,
multiclass_labels, metrics,
classification_indices,
- directories)
+ directories, example_ids)
if stats_iter > 1:
results_means_std = analyzebiclass_iter(
biclass_results, metrics, stats_iter, directory,
- labels_dictionary, data_base_name, nb_examples)
+ labels_dictionary, data_base_name, nb_examples, example_ids)
if nb_multiclass > 1:
results_means_std = analyze_iter_multiclass(multiclass_results, directory, stats_iter,
- metrics, data_base_name, nb_examples)
+ metrics, data_base_name, nb_examples, example_ids)
return results_means_std
diff --git a/multiview_platform/mono_multi_view_classifiers/utils/dataset.py b/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
index 991ee6a598daedd66db3de25cbebe0a27bccd165..1c3e59615df0fa1065c4c12f79cf77f717fbf4f6 100644
--- a/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
+++ b/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
@@ -66,7 +66,8 @@ class Dataset():
def __init__(self, views=None, labels=None, are_sparse=False,
file_name="dataset.hdf5", view_names=None, path="",
- hdf5_file=None, labels_names=None, is_temp=False):
+ hdf5_file=None, labels_names=None, is_temp=False,
+ example_ids=None):
self.is_temp = False
if hdf5_file is not None:
self.dataset=hdf5_file
@@ -104,6 +105,10 @@ class Dataset():
meta_data_grp.attrs["datasetLength"] = len(labels)
dataset_file.close()
self.update_hdf5_dataset(os.path.join(path, file_name))
+ if example_ids is not None:
+ self.example_ids = example_ids
+ else:
+ self.example_ids = [str(i) for i in range(labels.shape[0])]
def rm(self):
"""
@@ -146,6 +151,10 @@ class Dataset():
"""
self.nb_view = self.dataset["Metadata"].attrs["nbView"]
self.view_dict = self.get_view_dict()
+ if "example_ids" in self.dataset["Metadata"].keys():
+ self.example_ids = self.dataset["Metadata"]["example_ids"]
+ else:
+ self.example_ids = [str(i) for i in range(self.dataset["Labels"].shape[0])]
def get_nb_examples(self):
"""
diff --git a/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py b/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
index 1e3e6a83d3f9408452794ef1cbd8874c5c5c170b..4ba4e24b5dec76e638cbdbc4267d5a3e88c005dd 100644
--- a/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
+++ b/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
@@ -44,6 +44,7 @@ def get_plausible_db_hdf5(features, path, file_name, nb_class=3,
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
+ example_ids = ["exmaple_id_"+str(i) for i in range(nb_examples)]
views = []
view_names = []
are_sparse = []
@@ -72,10 +73,12 @@ def get_plausible_db_hdf5(features, path, file_name, nb_class=3,
view_names.append("ViewNumber" + str(view_index))
are_sparse.append(False)
+
+
dataset = Dataset(views=views, labels=labels,
labels_names=label_names, view_names=view_names,
are_sparse=are_sparse, file_name="plausible.hdf5",
- path=path)
+ path=path, example_ids=example_ids)
labels_dictionary = {0: "No", 1: "Yes"}
return dataset, labels_dictionary, "plausible"
elif nb_class >= 3:
@@ -114,7 +117,7 @@ def get_plausible_db_hdf5(features, path, file_name, nb_class=3,
labels_names=label_names, view_names=view_names,
are_sparse=are_sparse,
file_name="plausible.hdf5",
- path=path)
+ path=path, example_ids=example_ids)
labels_dictionary = {0: "No", 1: "Yes", 2: "Maybe"}
return dataset, labels_dictionary, "plausible"
diff --git a/multiview_platform/tests/test_ExecClassif.py b/multiview_platform/tests/test_ExecClassif.py
index abbcd77f933e6c9f49dc74213388551bbe85e61d..ad86757828f53a732ded8785cbf0f199bbbdbc9d 100644
--- a/multiview_platform/tests/test_ExecClassif.py
+++ b/multiview_platform/tests/test_ExecClassif.py
@@ -219,7 +219,7 @@ def fakeBenchmarkExec_monocore(dataset_var=1, a=4, args=1):
def fakegetResults(results, stats_iter, nb_multiclass,
benchmark_arguments_dictionaries, multi_class_labels, metrics,
classification_indices, directories, directory,
- labels_dictionary, nb_examples, nb_labels):
+ labels_dictionary, nb_examples, nb_labels, example_ids):
return 3
@@ -368,8 +368,7 @@ class Test_execOneBenchmark(unittest.TestCase):
1, 2, 1, 1,
2, 1, 21]),
exec_monoview_multicore=fakeExecMono,
- exec_multiview_multicore=fakeExecMulti,
- init_multiview_arguments=fakeInitMulti)
+ exec_multiview_multicore=fakeExecMulti,)
cls.assertEqual(flag, None)
cls.assertEqual(results ,
@@ -428,8 +427,7 @@ class Test_execOneBenchmark_multicore(unittest.TestCase):
flag=None,
labels=np.array([0, 1, 2, 3, 4, 2, 2, 12, 1, 2, 1, 1, 2, 1, 21]),
exec_monoview_multicore=fakeExecMono,
- exec_multiview_multicore=fakeExecMulti,
- init_multiview_arguments=fakeInitMulti)
+ exec_multiview_multicore=fakeExecMulti,)
cls.assertEqual(flag, None)
cls.assertEqual(results ,
diff --git a/multiview_platform/tests/test_ResultAnalysis.py b/multiview_platform/tests/test_ResultAnalysis.py
index bc739072790e9730058d7a9f916f66d512e7c31f..04531a2027d440c8019ab9771e689ec078db5657 100644
--- a/multiview_platform/tests/test_ResultAnalysis.py
+++ b/multiview_platform/tests/test_ResultAnalysis.py
@@ -1,56 +1,179 @@
-# import unittest
-# import numpy as np
-#
-# from ..mono_multi_view_classifiers import ResultAnalysis
-#
-#
-# class Test_getMetricsScoresBiclass(unittest.TestCase):
-#
-# @classmethod
-# def setUpClass(cls):
-# cls.metrics = [["accuracy_score"]]
-# cls.monoViewResults = [["", ["chicken_is_heaven", ["View0"], {"accuracy_score": [0.5,0.7]}]]]
-# cls.multiviewResults = [["Mumbo", {"":""}, {"accuracy_score":[0.6,0.8]}]]
-#
-# def test_simple(cls):
-# res = ResultAnalysis.getMetricsScoresBiclass(cls.metrics, cls.monoViewResults, cls.multiviewResults)
-# cls.assertIn("accuracy_score",res)
-# cls.assertEqual(type(res["accuracy_score"]), dict)
-# cls.assertEqual(res["accuracy_score"]["classifiers_names"], ["chicken_is_heaven-View0", "Mumbo"])
-# cls.assertEqual(res["accuracy_score"]["train_scores"], [0.5, 0.6])
-# cls.assertEqual(res["accuracy_score"]["test_scores"], [0.7, 0.8])
-#
-# def test_only_multiview(cls):
-# cls.monoViewResults = []
-# res = ResultAnalysis.getMetricsScoresBiclass(cls.metrics, cls.monoViewResults, cls.multiviewResults)
-# cls.assertIn("accuracy_score",res)
-# cls.assertEqual(type(res["accuracy_score"]), dict)
-# cls.assertEqual(res["accuracy_score"]["classifiers_names"], ["Mumbo"])
-# cls.assertEqual(res["accuracy_score"]["train_scores"], [0.6])
-# cls.assertEqual(res["accuracy_score"]["test_scores"], [0.8])
-#
-# def test_only_monoview(cls):
-# cls.multiviewResults = []
-# res = ResultAnalysis.getMetricsScoresBiclass(cls.metrics, cls.monoViewResults, cls.multiviewResults)
-# cls.assertIn("accuracy_score",res)
-# cls.assertEqual(type(res["accuracy_score"]), dict)
-# cls.assertEqual(res["accuracy_score"]["classifiers_names"], ["chicken_is_heaven-View0"])
-# cls.assertEqual(res["accuracy_score"]["train_scores"], [0.5])
-# cls.assertEqual(res["accuracy_score"]["test_scores"], [0.7])
-#
-#
-# class Test_getExampleErrorsBiclass(unittest.TestCase):
-#
-# @classmethod
-# def setUpClass(cls):
-# cls.usedBenchmarkArgumentDictionary = {"labels": np.array([0,1,1,-100,-100,0,1,1,-100])}
-# cls.monoViewResults = [["", ["chicken_is_heaven", ["View0"], {}, np.array([1,1,1,-100,-100,0,1,1,-100])]]]
-# cls.multiviewResults = [["Mumbo", {"":""}, {}, np.array([0,0,1,-100,-100,0,1,1,-100])]]
-#
-# def test_simple(cls):
-# res = ResultAnalysis.getExampleErrorsBiclass(cls.usedBenchmarkArgumentDictionary, cls.monoViewResults,
-# cls.multiviewResults)
-# cls.assertIn("chicken_is_heaven-View0", res)
-# cls.assertIn("Mumbo", res)
-# np.testing.assert_array_equal(res["Mumbo"], np.array([1,0,1,-100,-100,1,1,1,-100]))
-# np.testing.assert_array_equal(res["chicken_is_heaven-View0"], np.array([0,1,1,-100,-100,1,1,1,-100]))
+import unittest
+import numpy as np
+import pandas as pd
+import time
+
+from ..mono_multi_view_classifiers import result_analysis
+from ..mono_multi_view_classifiers.multiview.multiview_utils import MultiviewResult
+from ..mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult
+
+
+class Test_get_arguments(unittest.TestCase):
+
+ def setUp(self):
+ self.benchamrk_argument_dictionaries = [{"flag":"good_flag", "valid":True},
+ {"flag":"bad_flag", "valid":False}]
+
+ def test_benchmark_wanted(self):
+ argument_dict = result_analysis.get_arguments(self.benchamrk_argument_dictionaries, "good_flag")
+ self.assertTrue(argument_dict["valid"])
+
+
+class Test_get_metrics_scores_biclass(unittest.TestCase):
+
+
+ def test_simple(self):
+ metrics = [["accuracy_score"], ["f1_score"]]
+ results = [MonoviewResult(0,
+ "ada",
+ "0",
+ {"accuracy_score":[0.9, 0.95],
+ "f1_score":[0.91, 0.96]}
+ , "", "", "", "")]
+ metrics_scores = result_analysis.get_metrics_scores_biclass(metrics,
+ results)
+ self.assertIsInstance(metrics_scores, dict)
+ self.assertIsInstance(metrics_scores["accuracy_score"], pd.DataFrame)
+ np.testing.assert_array_equal(np.array(metrics_scores["accuracy_score"].loc["train"]), np.array([0.9]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["accuracy_score"].loc["test"]),
+ np.array([0.95]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["train"]),
+ np.array([0.91]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["test"]),
+ np.array([0.96]))
+ np.testing.assert_array_equal(np.array(metrics_scores["f1_score"].columns),
+ np.array(["ada-0"]))
+
+ def multiple_monoview_classifiers(self):
+ metrics = [["accuracy_score"], ["f1_score"]]
+ results = [MonoviewResult(0,
+ "ada",
+ "0",
+ {"accuracy_score": [0.9, 0.95],
+ "f1_score": [0.91, 0.96]}
+ , "", "", "", ""),
+ MonoviewResult(0,
+ "dt",
+ "1",
+ {"accuracy_score": [0.8, 0.85],
+ "f1_score": [0.81, 0.86]}
+ , "", "", "", "")
+ ]
+ metrics_scores = result_analysis.get_metrics_scores_biclass(metrics,
+ results)
+ self.assertIsInstance(metrics_scores, dict)
+ self.assertIsInstance(metrics_scores["accuracy_score"], pd.DataFrame)
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["accuracy_score"].loc["train"]),
+ np.array([0.9, 0.8]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["accuracy_score"].loc["test"]),
+ np.array([0.95, 0.85]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["train"]),
+ np.array([0.91, 0.81]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["test"]),
+ np.array([0.96, 0.86]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].columns),
+ np.array(["ada-0", "dt-1"]))
+
+ def mutiview_result(self):
+ metrics = [["accuracy_score"], ["f1_score"]]
+ results = [MultiviewResult("mv", "", {"accuracy_score": [0.7, 0.75],
+ "f1_score": [0.71, 0.76]}, "", ""),
+ MonoviewResult(0,
+ "dt",
+ "1",
+ {"accuracy_score": [0.8, 0.85],
+ "f1_score": [0.81, 0.86]}
+ , "", "", "", "")
+ ]
+ metrics_scores = result_analysis.get_metrics_scores_biclass(metrics,
+ results)
+ self.assertIsInstance(metrics_scores, dict)
+ self.assertIsInstance(metrics_scores["accuracy_score"], pd.DataFrame)
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["accuracy_score"].loc["train"]),
+ np.array([0.7, 0.8]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["accuracy_score"].loc["test"]),
+ np.array([0.75, 0.85]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["train"]),
+ np.array([0.71, 0.81]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].loc["test"]),
+ np.array([0.76, 0.86]))
+ np.testing.assert_array_equal(
+ np.array(metrics_scores["f1_score"].columns),
+ np.array(["mv", "dt-1"]))
+
+class Test_get_example_errors_biclass(unittest.TestCase):
+
+ def test_simple(self):
+ ground_truth = np.array([0,1,0,1,0,1,0,1, -100])
+ results = [MultiviewResult("mv", "", {"accuracy_score": [0.7, 0.75],
+ "f1_score": [0.71, 0.76]},
+ np.array([0,0,0,0,1,1,1,1,1]),
+ ""),
+ MonoviewResult(0,
+ "dt",
+ "1",
+ {"accuracy_score": [0.8, 0.85],
+ "f1_score": [0.81, 0.86]}
+ , np.array([0,0,1,1,0,0,1,1,0]), "", "", "")
+ ]
+ example_errors = result_analysis.get_example_errors_biclass(ground_truth,
+ results)
+ self.assertIsInstance(example_errors, dict)
+ np.testing.assert_array_equal(example_errors["mv"],
+ np.array([1,0,1,0,0,1,0,1,-100]))
+ np.testing.assert_array_equal(example_errors["dt-1"],
+ np.array([1, 0, 0, 1, 1, 0, 0, 1,-100]))
+
+
+class Test_init_plot(unittest.TestCase):
+
+ def test_simple(self):
+ results = []
+ metric_name = "acc"
+ data = np.random.RandomState(42).uniform(0,1,(2,2))
+ metric_dataframe = pd.DataFrame(index=["train", "test"], columns=["dt-1", "mv"], data=data)
+ directory = "dir"
+ database_name = 'db'
+ labels_names = ['lb1', "lb2"]
+ train, test, classifier_names, \
+ file_name, nb_results, results = result_analysis.init_plot(results,
+ metric_name,
+ metric_dataframe,
+ directory,
+ database_name,
+ labels_names)
+ self.assertEqual(file_name, "dir"+time.strftime(
+ "%Y_%m_%d-%H_%M_%S")+"-db-lb1_vs_lb2-acc")
+ np.testing.assert_array_equal(train, data[0,:])
+ np.testing.assert_array_equal(test, data[1, :])
+ np.testing.assert_array_equal(classifier_names, np.array(["dt-1", "mv"]))
+ self.assertEqual(nb_results, 2)
+ self.assertEqual(results, [["dt-1", "acc", data[1,0], 0], ["mv", "acc", data[1,1], 0]])
+
+class Test_gen_error_data(unittest.TestCase):
+
+ def test_simple(self):
+ random_state = np.random.RandomState(42)
+ ada_data = random_state.randint(0,2,size=7)
+ mv_data = random_state.randint(0, 2, size=7)
+ example_errors = {"ada-1": ada_data,
+ "mv": mv_data}
+ nb_classifiers, nb_examples, classifiers_names, \
+ data_2d, error_on_examples = result_analysis.gen_error_data(example_errors)
+ self.assertEqual(nb_classifiers, 2)
+ self.assertEqual(nb_examples, 7)
+ self.assertEqual(classifiers_names, ["ada-1", "mv"])
+ np.testing.assert_array_equal(data_2d, np.array([ada_data, mv_data]).transpose())
+ np.testing.assert_array_equal(error_on_examples, -1*(ada_data+mv_data)/nb_classifiers)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index a99989254c35ff3dfa00bfb97997f888a40caa89..51d86630547f58db44af3f661421c6b5a1edb571 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,4 +14,5 @@ pandas==0.23.3
m2r==0.2.1
docutils==0.12
pyyaml==3.12
-cvxopt==1.2.0
\ No newline at end of file
+cvxopt==1.2.0
+plotly==4.2.1
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 3bc02d0f1fa3a6d2d666ac213262ba4f34a77a6d..885aa563ed0c272941262dd22fa4ef9bd33553f8 100644
--- a/setup.py
+++ b/setup.py
@@ -55,7 +55,7 @@ def setup_package():
install_requires=['numpy>=1.16', 'scipy>=0.16','scikit-learn==0.19',
'matplotlib', 'h5py', 'joblib',
'pandas', 'm2r', 'pyyaml', 'pyscm @ git+https://github.com/aldro61/pyscm',
- 'cvxopt'],
+ 'cvxopt', 'plotly==4.2.1'],
# Il est d'usage de mettre quelques metadata à propos de sa lib
# Pour que les robots puissent facilement la classer.