Skip to content
Snippets Groups Projects
Commit 243ec127 authored by Baptiste Bauvin's avatar Baptiste Bauvin
Browse files

Merge branch 'develop'

parents 15171bde 615bd532
Branches
Tags
No related merge requests found
Pipeline #4835 passed
Showing
with 222 additions and 17 deletions
...@@ -24,3 +24,5 @@ multiview_platform/examples/results/example_4/* ...@@ -24,3 +24,5 @@ multiview_platform/examples/results/example_4/*
multiview_platform/examples/results/example_5/* multiview_platform/examples/results/example_5/*
multiview_platform/html_cov/ multiview_platform/html_cov/
multiview_platform/.coverage* multiview_platform/.coverage*
.coverage*
htmlcov/
\ No newline at end of file
...@@ -56,3 +56,4 @@ pages: ...@@ -56,3 +56,4 @@ pages:
artifacts: artifacts:
paths: paths:
- public - public
...@@ -47,7 +47,7 @@ def get_example_errors(groud_truth, results): ...@@ -47,7 +47,7 @@ def get_example_errors(groud_truth, results):
def publish_example_errors(example_errors, directory, databaseName, def publish_example_errors(example_errors, directory, databaseName,
labels_names, example_ids, labels): labels_names, example_ids, labels): # pragma: no cover
logging.debug("Start:\t Label analysis figure generation") logging.debug("Start:\t Label analysis figure generation")
base_file_name = os.path.join(directory, databaseName + "-" ) base_file_name = os.path.join(directory, databaseName + "-" )
...@@ -70,7 +70,7 @@ def publish_example_errors(example_errors, directory, databaseName, ...@@ -70,7 +70,7 @@ def publish_example_errors(example_errors, directory, databaseName,
def publish_all_example_errors(iter_results, directory, def publish_all_example_errors(iter_results, directory,
stats_iter, stats_iter,
example_ids, labels): example_ids, labels): # pragma: no cover
logging.debug( logging.debug(
"Start:\t Global label analysis figure generation") "Start:\t Global label analysis figure generation")
...@@ -132,11 +132,7 @@ def gen_error_data(example_errors): ...@@ -132,11 +132,7 @@ def gen_error_data(example_errors):
data_2d = np.zeros((nb_examples, nb_classifiers)) data_2d = np.zeros((nb_examples, nb_classifiers))
for classifierIndex, (classifier_name, error_on_examples) in enumerate( for classifierIndex, (classifier_name, error_on_examples) in enumerate(
example_errors.items()): example_errors.items()):
try:
data_2d[:, classifierIndex] = error_on_examples data_2d[:, classifierIndex] = error_on_examples
except:
import pdb;
pdb.set_trace()
error_on_examples = np.sum(data_2d, axis=1) / nb_classifiers error_on_examples = np.sum(data_2d, axis=1) / nb_classifiers
return nb_classifiers, nb_examples, classifiers_names, data_2d, error_on_examples return nb_classifiers, nb_examples, classifiers_names, data_2d, error_on_examples
...@@ -267,7 +263,7 @@ def plot_errors_bar(error_on_examples, nb_examples, file_name, ...@@ -267,7 +263,7 @@ def plot_errors_bar(error_on_examples, nb_examples, file_name,
def iter_cmap(statsIter): def iter_cmap(statsIter): # pragma: no cover
r"""Used to generate a colormap that will have a tick for each iteration : the whiter the better. r"""Used to generate a colormap that will have a tick for each iteration : the whiter the better.
Parameters Parameters
......
...@@ -8,7 +8,7 @@ from .error_analysis import get_example_errors, publish_example_errors, publish_ ...@@ -8,7 +8,7 @@ from .error_analysis import get_example_errors, publish_example_errors, publish_
from .feature_importances import get_feature_importances, publish_feature_importances from .feature_importances import get_feature_importances, publish_feature_importances
def analyze(results, stats_iter, benchmark_argument_dictionaries, def analyze(results, stats_iter, benchmark_argument_dictionaries,
metrics, directory, example_ids, labels): metrics, directory, example_ids, labels): # pragma: no cover
"""Used to analyze the results of the previous benchmarks""" """Used to analyze the results of the previous benchmarks"""
data_base_name = benchmark_argument_dictionaries[0]["args"]["name"] data_base_name = benchmark_argument_dictionaries[0]["args"]["name"]
...@@ -105,7 +105,7 @@ def analyze_iterations(results, benchmark_argument_dictionaries, stats_iter, ...@@ -105,7 +105,7 @@ def analyze_iterations(results, benchmark_argument_dictionaries, stats_iter,
def analyze_all(iter_results, stats_iter, directory, data_base_name, def analyze_all(iter_results, stats_iter, directory, data_base_name,
example_ids, label_names): example_ids, label_names): # pragma: no cover
"""Used to format the results in order to plot the mean results on """Used to format the results in order to plot the mean results on
the iterations""" the iterations"""
metrics_analysis, class_metrics_analysis, error_analysis, feature_importances, \ metrics_analysis, class_metrics_analysis, error_analysis, feature_importances, \
......
...@@ -38,7 +38,7 @@ def get_feature_importances(result, feature_names=None): ...@@ -38,7 +38,7 @@ def get_feature_importances(result, feature_names=None):
return feature_importances return feature_importances
def publish_feature_importances(feature_importances, directory, database_name, def publish_feature_importances(feature_importances, directory, database_name,
feature_stds=None): feature_stds=None): # pragma: no cover
for view_name, feature_importance in feature_importances.items(): for view_name, feature_importance in feature_importances.items():
if not os.path.exists(os.path.join(directory, "feature_importances")): if not os.path.exists(os.path.join(directory, "feature_importances")):
os.mkdir(os.path.join(directory, "feature_importances")) os.mkdir(os.path.join(directory, "feature_importances"))
......
...@@ -67,7 +67,7 @@ def get_metrics_scores(metrics, results, label_names): ...@@ -67,7 +67,7 @@ def get_metrics_scores(metrics, results, label_names):
def publish_metrics_graphs(metrics_scores, directory, database_name, def publish_metrics_graphs(metrics_scores, directory, database_name,
labels_names, class_metric_scores): labels_names, class_metric_scores): # pragma: no cover
r"""Used to sort the results (names and both scores) in descending test r"""Used to sort the results (names and both scores) in descending test
score order. score order.
...@@ -114,7 +114,7 @@ def publish_metrics_graphs(metrics_scores, directory, database_name, ...@@ -114,7 +114,7 @@ def publish_metrics_graphs(metrics_scores, directory, database_name,
def publish_all_metrics_scores(iter_results, class_iter_results, directory, def publish_all_metrics_scores(iter_results, class_iter_results, directory,
data_base_name, stats_iter, label_names, data_base_name, stats_iter, label_names,
min_size=10): min_size=10): # pragma: no cover
results = [] results = []
secure_file_path(os.path.join(directory, "a")) secure_file_path(os.path.join(directory, "a"))
......
import unittest
import numpy as np
import pandas as pd
from multiview_platform.mono_multi_view_classifiers.result_analysis import duration_analysis
class FakeClassifierResult:
def __init__(self, i=0):
self.i=i
if i == 0:
self.hps_duration = 10
self.fit_duration = 12
self.pred_duration = 15
else:
self.hps_duration = 1
self.fit_duration = 2
self.pred_duration = 5
def get_classifier_name(self):
if self.i == 0:
return 'test1'
else:
return 'test2'
class Test_get_duration(unittest.TestCase):
def test_simple(self):
results = [FakeClassifierResult(), FakeClassifierResult(i=1)]
durs = duration_analysis.get_duration(results)
pd.testing.assert_frame_equal(durs,
pd.DataFrame(index=['test1', 'test2'],
columns=['hps', 'fit', 'pred'],
data=np.array([np.array([10,12,15]),
np.array([1,2,5])]),
dtype=object))
import unittest import unittest
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import os
from multiview_platform.mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult from multiview_platform.mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult
from multiview_platform.mono_multi_view_classifiers.multiview.multiview_utils import MultiviewResult from multiview_platform.mono_multi_view_classifiers.multiview.multiview_utils import MultiviewResult
from multiview_platform.mono_multi_view_classifiers.result_analysis.execution import format_previous_results, get_arguments from multiview_platform.mono_multi_view_classifiers.result_analysis.execution import format_previous_results, get_arguments, analyze_iterations
from multiview_platform.tests.utils import rm_tmp, tmp_path, test_dataset
class FakeClassifierResult:
def __init__(self, i=1):
self.classifier_name='test'+str(i)
self.full_labels_pred = np.array([0,1,1,2,1])
self.hps_duration=i
self.fit_duration=i
self.pred_duration=i
def get_classifier_name(self):
return self.classifier_name
class Test_format_previous_results(unittest.TestCase): class Test_format_previous_results(unittest.TestCase):
...@@ -82,3 +97,43 @@ class Test_get_arguments(unittest.TestCase): ...@@ -82,3 +97,43 @@ class Test_get_arguments(unittest.TestCase):
def test_benchmark_wanted(self): def test_benchmark_wanted(self):
argument_dict = get_arguments(self.benchamrk_argument_dictionaries, "good_flag") argument_dict = get_arguments(self.benchamrk_argument_dictionaries, "good_flag")
self.assertTrue(argument_dict["valid"]) self.assertTrue(argument_dict["valid"])
class Test_analyze_iterations(unittest.TestCase):
@classmethod
def setUpClass(cls):
os.mkdir(tmp_path)
cls.results = [[0, [FakeClassifierResult(), FakeClassifierResult(i=2)], []], [1, [FakeClassifierResult(), FakeClassifierResult(i=2)], []]]
cls.benchmark_argument_dictionaries = [{"labels_dictionary":{0:"zero",1:"one",2:"two"}, "flag":0, "directory":tmp_path, "args":{"name":"test_dataset"}},{"labels_dictionary":{0:"zero",1:"one",2:"two"}, "flag":1, "directory":tmp_path, "args":{"name":"test_dataset"}} ]
cls.stats_iter = 2
cls.metrics = {}
cls.example_ids = ['ex1', 'ex5','ex4','ex3','ex2',]
cls.labels = np.array([0,1,2,1,1])
@classmethod
def tearDownClass(cls):
rm_tmp()
def test_simple(self):
analysis = analyze_iterations(self.results,
self.benchmark_argument_dictionaries,
self.stats_iter,
self.metrics,
self.example_ids,
self.labels)
res, iter_res, tracebacks, labels_names = analysis
self.assertEqual(labels_names, ['zero', 'one', 'two'])
self.assertEqual(iter_res['class_metrics_scores'], [{}, {}])
pd.testing.assert_frame_equal(iter_res['durations'][0], pd.DataFrame(index=['test1','test2'],
columns=['hps', 'fit', 'pred'],
data=np.array([1,1,1,2,2,2]).reshape((2,3)), dtype=object))
np.testing.assert_array_equal(iter_res['example_errors'][0]['test1'], np.array([1, 1, 0, 0, 1]))
self.assertEqual(iter_res["feature_importances"], [{},{}])
np.testing.assert_array_equal(iter_res['labels'], np.array([0, 1, 2, 1, 1]))
self.assertEqual(iter_res['metrics_scores'], [{},{}])
import unittest
import numpy as np
import pandas as pd
from multiview_platform.mono_multi_view_classifiers.result_analysis import feature_importances
from multiview_platform.mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult
class FakeClassifier:
def __init__(self, i=0):
self.feature_importances_ = [i, i+1]
class FakeClassifierResult(MonoviewResult):
def __init__(self, i=0):
self.i=i
self.hps_duration = i*10
self.fit_duration = (i+2)*10
self.pred_duration = (i+5)*10
self.clf = FakeClassifier(i)
self.view_name = 'testview'+str(i)
self.classifier_name = "test"+str(i)
def get_classifier_name(self):
return self.classifier_name
class Test_get_duration(unittest.TestCase):
def test_simple(self):
results = [FakeClassifierResult(), FakeClassifierResult(i=1)]
feat_importance = feature_importances.get_feature_importances(results)
pd.testing.assert_frame_equal(feat_importance["testview1"],
pd.DataFrame(index=None,columns=['test1'],
data=np.array([1,2]).reshape((2,1)),
))
\ No newline at end of file
...@@ -6,7 +6,7 @@ import os ...@@ -6,7 +6,7 @@ import os
from multiview_platform.mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult from multiview_platform.mono_multi_view_classifiers.monoview.monoview_utils import MonoviewResult
from multiview_platform.mono_multi_view_classifiers.multiview.multiview_utils import MultiviewResult from multiview_platform.mono_multi_view_classifiers.multiview.multiview_utils import MultiviewResult
from multiview_platform.mono_multi_view_classifiers.result_analysis.metric_analysis import get_metrics_scores, init_plot from multiview_platform.mono_multi_view_classifiers.result_analysis.metric_analysis import get_metrics_scores, init_plot, get_fig_size, sort_by_test_score
class Test_get_metrics_scores(unittest.TestCase): class Test_get_metrics_scores(unittest.TestCase):
...@@ -151,3 +151,30 @@ class Test_init_plot(unittest.TestCase): ...@@ -151,3 +151,30 @@ class Test_init_plot(unittest.TestCase):
self.assertEqual(nb_results, 2) self.assertEqual(nb_results, 2)
self.assertEqual(results, [["dt-1", "acc", data[1,0], 0.0, data[1,0]], self.assertEqual(results, [["dt-1", "acc", data[1,0], 0.0, data[1,0]],
["mv", "acc", data[1,1], 0.0, data[1,1]]]) ["mv", "acc", data[1,1], 0.0, data[1,1]]])
class Test_small_func(unittest.TestCase):
def test_fig_size(self):
kw, width = get_fig_size(5)
self.assertEqual(kw, {"figsize":(15,5)})
self.assertEqual(width, 0.35)
kw, width = get_fig_size(100)
self.assertEqual(kw, {"figsize": (100, 100/3)})
self.assertEqual(width, 0.35)
def test_sort_by_test_scores(self):
train_scores = np.array([1,2,3,4])
test_scores = np.array([4, 3, 2, 1])
train_STDs = np.array([1, 2, 3, 4])
test_STDs = np.array([1, 2, 3, 4])
names = np.array(['1', '2', '3', '4'])
sorted_names, sorted_train_scores, \
sorted_test_scores, sorted_train_STDs, \
sorted_test_STDs = sort_by_test_score(train_scores, test_scores,
names, train_STDs, test_STDs)
np.testing.assert_array_equal(sorted_names, np.array(['4', '3', '2', '1']))
np.testing.assert_array_equal(sorted_test_scores, [1, 2, 3, 4])
np.testing.assert_array_equal(sorted_test_STDs, [4, 3, 2, 1])
np.testing.assert_array_equal(sorted_train_scores, [4, 3, 2, 1])
np.testing.assert_array_equal(sorted_train_STDs, [4, 3, 2, 1])
\ No newline at end of file
import unittest
import numpy as np
import pandas as pd
import os
from multiview_platform.mono_multi_view_classifiers.result_analysis import tracebacks_analysis
from multiview_platform.tests.utils import tmp_path, rm_tmp
class FakeClassifierResult:
def __init__(self, i=0):
self.i=i
if i == 0:
self.hps_duration = 10
self.fit_duration = 12
self.pred_duration = 15
else:
self.hps_duration = 1
self.fit_duration = 2
self.pred_duration = 5
def get_classifier_name(self):
if self.i == 0:
return 'test1'
else:
return 'test2'
class Test_funcs(unittest.TestCase):
@classmethod
def setUpClass(cls):
os.mkdir(tmp_path)
cls.res_file = open(os.path.join(tmp_path,"tmp.txt"), "w")
@classmethod
def tearDownClass(cls):
rm_tmp()
def test_save_dict_to_text(self):
keys = tracebacks_analysis.save_dict_to_text({"a":"i", "b":"j"}, self.res_file)
self.res_file.close()
self.assertEqual(list(keys),["a", "b"])
with open(os.path.join(tmp_path,"tmp.txt"), 'r') as res_file:
self.assertEqual(res_file.read(), 'Failed algorithms : \n\ta,\n\tb.\n\n\na\n\ni\n\n\nb\n\nj\n\n\n')
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment