diff --git a/.gitignore b/.gitignore
index 3c5d0bd4056c9d6b997252b9264b11cda228d65d..4d1a1ab0105b41dce257f832bde652608a075a96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ TODO
ipynb/.ipynb_checkpoints/**
docs/source/monomulti/.ipynb_checkpoints/**
results/**
+data/**
Data/**
multiview_platform/MonoMultiviewClassifiers/Results/*
multiview_platform/tests/temp_tests/**
diff --git a/config_files/config.yml b/config_files/config.yml
index 318ef51d36e0a6d7fe96dc0201c48c638fb12bbd..33c5469e8a209f553a1cd3eb726b6a3d3add7499 100644
--- a/config_files/config.yml
+++ b/config_files/config.yml
@@ -4,7 +4,7 @@ Base :
name: ["Plausible"]
label: "_"
type: ".hdf5"
- views: ["all"]
+ views:
pathf: "../data/"
nice: 0
random_state: 42
@@ -21,9 +21,9 @@ Classification:
split: 0.8
nb_folds: 2
nb_class: 2
- classes: ["yes", "no"]
- type: ["multiview",]
- algos_monoview: ["all"]
+ classes:
+ type: ["multiview"]
+ algos_monoview: ["decision_tree", "adaboost"]
algos_multiview: ["all"]
stats_iter: 2
metrics: ["accuracy_score", "f1_score"]
@@ -170,6 +170,41 @@ double_fault_fusion:
splitter: ["best"]
difficulty_fusion:
+ classifier_names: ["decison_tree"]
+ classifier_configs:
+ decision_tree:
+ max_depth: [1]
+ criterion: ["gini"]
+ splitter: ["best"]
+
+scm_late_fusion:
+ classifier_names: ["decison_tree"]
+ p: 0.1
+ max_rules: 10
+ model_type: 'conjunction'
+ classifier_configs:
+ decision_tree:
+ max_depth: [1]
+ criterion: ["gini"]
+ splitter: ["best"]
+
+majority_voting_fusion:
+ classifier_names: ["decison_tree"]
+ classifier_configs:
+ decision_tree:
+ max_depth: [1]
+ criterion: ["gini"]
+ splitter: ["best"]
+
+bayesian_inference_fusion:
+ classifier_names: ["decison_tree"]
+ classifier_configs:
+ decision_tree:
+ max_depth: [1]
+ criterion: ["gini"]
+ splitter: ["best"]
+
+weighted_linear_late_fusion:
classifier_names: ["decison_tree"]
classifier_configs:
decision_tree:
diff --git a/multiview_platform/__init__.py b/multiview_platform/__init__.py
index f91d950e4c0743ea01764c2d899fb6a4c0966e09..3ca9940baba1a9b6ade351a85567f2805a53676c 100644
--- a/multiview_platform/__init__.py
+++ b/multiview_platform/__init__.py
@@ -2,4 +2,4 @@
__version__ = "0.0.0.0"
-from . import mono_multi_view_classifiers, tests, execute, versions
+from . import mono_multi_view_classifiers, execute, versions
diff --git a/multiview_platform/declare_classifier.py b/multiview_platform/declare_classifier.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d83f06af5bc53bc01c11e949ff96cdb6be7b7bf
--- /dev/null
+++ b/multiview_platform/declare_classifier.py
@@ -0,0 +1,53 @@
+import os
+import importlib
+import inspect
+
+
+class ClassierMakerMultiViewPlatform():
+ _benchmark = {"monoview":
+ {"path_classifier": 'multiview_platform/mono_multi_view_classifier/monoview_classifiers'},
+ "multiview":
+ {"path_classifier_multi": 'multiview_platform/mono_multi_view_classifier/multiview_classifier'}}
+
+
+ def __init__(self, classifier_names, classifier_modules=None, classifier_files=None, mod='monoview'):
+ if classifier_files is None and classifier_names.size != classifier_modules.size:
+ raise ValueError("attr classifier_names and classifier_modules should have same size")
+ if classifier_modules is None and classifier_names.size != classifier_files.size:
+ raise ValueError("attr classifier_names and classifier_files should have same size")
+
+ if classifier_files is None:
+ for classifier, module in zip(classifier_names, classifier_modules):
+ my_instance, my_module = self._check_classifier_install
+
+ self._create_class(my_instance, my_module)
+
+
+ def _check_classifier_install(self, classifier, module):
+ try:
+ my_module = importlib.import_module(module)
+ except Exception:
+ raise("the module %d can't be imported" % module)
+ try:
+ my_instance = getattr(my_module, classifier)
+ except AttributeError:
+ raise AttributeError("The class %d is not in %d" % classifier %module)
+ return my_instance, my_module
+
+ def _create_class(self, classifier, module):
+ if mod.startswith('monoview'):
+ directory = self._benchmark[mod]["path_classifier"]
+
+
+ def _get_module_name(self, mymodule):
+ for name in dir(mymodule):
+ att = getattr(mymodule, name)
+ try:
+ getattr(att, "__module__")
+ if att.__module__.startswith(mymodule.__name__):
+ if inspect.isclass(att):
+ if att == name:
+ return name
+ except Exception:
+ return None
+ return None
\ No newline at end of file
diff --git a/multiview_platform/mono_multi_view_classifiers/exec_classif.py b/multiview_platform/mono_multi_view_classifiers/exec_classif.py
index cba482d1ab1051ee28e8702bf12f550e04043a71..2adbba0dadfc8f75544c2ececd36dcdac488a1e1 100644
--- a/multiview_platform/mono_multi_view_classifiers/exec_classif.py
+++ b/multiview_platform/mono_multi_view_classifiers/exec_classif.py
@@ -79,32 +79,32 @@ def init_benchmark(cl_type, monoview_algos, multiview_algos, args):
return benchmark
-def gen_views_dictionnary(dataset_var, views):
- r"""Used to generate a dictionary mapping a view name (key) to it's index in the dataset (value).
-
- Parameters
- ----------
- dataset_var : `h5py` dataset file
- The full dataset on which the benchmark will be done
- views : List of strings
- Names of the selected views on which the banchmark will be done
-
- Returns
- -------
- viewDictionary : Dictionary
- Dictionary mapping the view names totheir indexin the full dataset.
- """
- datasets_names = dataset_var.keys()
- views_dictionary = {}
- for dataset_name in datasets_names:
- if dataset_name[:4] == "View":
- view_name = dataset_var.get(dataset_name).attrs["name"]
- if type(view_name) == bytes:
- view_name = view_name.decode("utf-8")
- if view_name in views:
- views_dictionary[view_name] = int(dataset_name[4:])
-
- return views_dictionary
+# def gen_views_dictionnary(dataset_var, views):
+# r"""Used to generate a dictionary mapping a view name (key) to it's index in the dataset (value).
+#
+# Parameters
+# ----------
+# dataset_var : `h5py` dataset file
+# The full dataset on which the benchmark will be done
+# views : List of strings
+# Names of the selected views on which the banchmark will be done
+#
+# Returns
+# -------
+# viewDictionary : Dictionary
+# Dictionary mapping the view names totheir indexin the full dataset.
+# """
+# datasets_names = dataset_var.get_view_dict().keys()
+# views_dictionary = {}
+# for dataset_name in datasets_names:
+# if dataset_name[:4] == "View":
+# view_name = dataset_var.get(dataset_name).attrs["name"]
+# if type(view_name) == bytes:
+# view_name = view_name.decode("utf-8")
+# if view_name in views:
+# views_dictionary[view_name] = int(dataset_name[4:])
+#
+# return views_dictionary
def init_argument_dictionaries(benchmark, views_dictionary,
@@ -312,7 +312,7 @@ def gen_multiple_args_dictionnaries(nb_class, kwargs_init, classifier,
return args_dictionnaries
-def init_kwargs(args, classifiers_names):
+def init_kwargs(args, classifiers_names, framework="monoview"):
r"""Used to init kwargs thanks to a function in each monoview classifier package.
Parameters
@@ -330,42 +330,44 @@ def init_kwargs(args, classifiers_names):
For example, for Adaboost, the KWARGS will be `{"n_estimators":<value>, "base_estimator":<value>}`"""
logging.debug("Start:\t Initializing monoview classifiers arguments")
- monoview_kwargs = {}
+ kwargs = {}
for classifiers_name in classifiers_names:
try:
- getattr(monoview_classifiers, classifiers_name)
+ if framework=="monoview":
+ getattr(monoview_classifiers, classifiers_name)
+ else:
+ getattr(multiview_classifiers, classifiers_name)
except AttributeError:
raise AttributeError(
classifiers_name + " is not implemented in monoview_classifiers, "
"please specify the name of the file in monoview_classifiers")
- monoview_kwargs[
- classifiers_name] = args[classifiers_name]
+ kwargs[classifiers_name] = args[classifiers_name]
logging.debug("Done:\t Initializing monoview classifiers arguments")
- return monoview_kwargs
+ return kwargs
def init_kwargs_func(args, benchmark):
monoview_kwargs = init_kwargs(args, benchmark["monoview"])
- multiview_kwargs = init_kwargs(args, benchmark["multiview"])
+ multiview_kwargs = init_kwargs(args, benchmark["multiview"], framework="multiview")
kwargs = {"monoview":monoview_kwargs, "multiview":multiview_kwargs}
return kwargs
-def init_multiview_kwargs(args, classifiers_names):
- logging.debug("Start:\t Initializing multiview classifiers arguments")
- multiview_kwargs = {}
- for classifiers_name in classifiers_names:
- try:
- getattr(multiview_classifiers, classifiers_name)
- except AttributeError:
- raise AttributeError(
- classifiers_name + " is not implemented in mutliview_classifiers, "
- "please specify the name of the coressponding .py "
- "file in mutliview_classifiers")
- multiview_kwargs[classifiers_name] = args[classifiers_name]
- logging.debug("Done:\t Initializing multiview classifiers arguments")
- return multiview_kwargs
+# def init_multiview_kwargs(args, classifiers_names):
+# logging.debug("Start:\t Initializing multiview classifiers arguments")
+# multiview_kwargs = {}
+# for classifiers_name in classifiers_names:
+# try:
+# getattr(multiview_classifiers, classifiers_name)
+# except AttributeError:
+# raise AttributeError(
+# classifiers_name + " is not implemented in mutliview_classifiers, "
+# "please specify the name of the coressponding .py "
+# "file in mutliview_classifiers")
+# multiview_kwargs[classifiers_name] = args[classifiers_name]
+# logging.debug("Done:\t Initializing multiview classifiers arguments")
+# return multiview_kwargs
def init_multiview_arguments(args, benchmark, views, views_indices,
@@ -572,7 +574,7 @@ def exec_one_benchmark_mono_core(dataset_var=None, labels_dictionary=None,
labels_dictionary, k_folds)
logging.debug("Start:\t monoview benchmark")
for arguments in argument_dictionaries["monoview"]:
- X = dataset_var.get("View" + str(arguments["view_index"]))
+ X = dataset_var.get_v(arguments["view_index"])
Y = labels
results_monoview += [
exec_monoview(directory, X, Y, args["Base"]["name"], labels_names,
@@ -681,7 +683,7 @@ def exec_benchmark(nb_cores, stats_iter, nb_multiclass,
# Do everything with flagging
nb_examples = len(classification_indices[0][0]) + len(
classification_indices[0][1])
- multiclass_ground_truth = dataset_var.get("Labels").value
+ multiclass_ground_truth = dataset_var.get_labels()
logging.debug("Start:\t Analyzing predictions")
results_mean_stds = get_results(results, stats_iter, nb_multiclass,
benchmark_arguments_dictionaries,
@@ -755,7 +757,7 @@ def exec_classif(arguments):
views, views_indices, all_views = execution.init_views(dataset_var, args["Base"]["views"])
- views_dictionary = gen_views_dictionnary(dataset_var, views)
+ views_dictionary = dataset_var.get_view_dict()
nb_views = len(views)
nb_class = dataset_var.get_nb_class()
diff --git a/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py b/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
index 4492cdff468546bfa28e1332528f4046ec7636aa..9bf64f7a67b4f460bfe43a88103ea0f03fbada13 100644
--- a/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
+++ b/multiview_platform/mono_multi_view_classifiers/monoview/exec_classif_mono_view.py
@@ -16,7 +16,7 @@ from . import monoview_utils
from .analyze_result import execute
# Import own modules
from .. import monoview_classifiers
-from ..utils.dataset import get_value, extract_subset
+from ..utils.dataset import get_value, extract_subset, Dataset
from ..utils import hyper_parameter_search
# Author-Info
@@ -33,31 +33,32 @@ def exec_monoview_multicore(directory, name, labels_names, classification_indice
hyper_param_search="randomized_search",
metrics=[["accuracy_score", None]], n_iter=30,
**args):
- dataset_var = h5py.File(path + name + str(dataset_file_index) + ".hdf5", "r")
- neededViewIndex = args["viewIndex"]
- X = dataset_var.get("View" + str(neededViewIndex))
+ dataset_var = Dataset(hdf5_file=h5py.File(path + name + str(dataset_file_index) + ".hdf5", "r"))
+ neededViewIndex = args["view_index"]
+ X = dataset_var.get_v(neededViewIndex)
Y = labels
return exec_monoview(directory, X, Y, name, labels_names,
classification_indices, k_folds, 1, database_type, path,
random_state, hyper_param_search=hyper_param_search,
-
- metrics=metrics, n_iter=n_iter, **args)
+ metrics=metrics, n_iter=n_iter,
+ view_name=dataset_var.get_view_name(args["view_index"]),
+ **args)
def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
KFolds, nbCores, databaseType, path,
randomState, hyper_param_search="randomized_search",
- metrics=[["accuracy_score", None]], nIter=30, **args):
+ metrics=[["accuracy_score", None]], nIter=30, view_name="", **args):
logging.debug("Start:\t Loading data")
kwargs, \
t_start, \
feat, \
- CL_type, \
+ classifier_name, \
X, \
learningRate, \
labelsString, \
outputFileName = initConstants(args, X, classificationIndices, labels_names,
- name, directory)
+ name, directory, view_name)
logging.debug("Done:\t Loading data")
logging.debug(
@@ -65,7 +66,7 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
feat) + " train ratio:"
+ str(learningRate) + ", CrossValidation k-folds: " + str(
KFolds.n_splits) + ", cores:"
- + str(nbCores) + ", algorithm : " + CL_type)
+ + str(nbCores) + ", algorithm : " + classifier_name)
logging.debug("Start:\t Determine Train/Test split")
X_train, y_train, X_test, y_test, X_test_multiclass = init_train_test(X, Y,
@@ -78,10 +79,10 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
logging.debug("Done:\t Determine Train/Test split")
logging.debug("Start:\t Generate classifier args")
- classifierModule = getattr(monoview_classifiers, CL_type)
+ classifierModule = getattr(monoview_classifiers, classifier_name)
classifier_class_name = classifierModule.classifier_class_name
- clKWARGS, testFoldsPreds = getHPs(classifierModule, hyper_parameter_search,
- nIter, CL_type, classifier_class_name,
+ clKWARGS, testFoldsPreds = getHPs(classifierModule, hyper_param_search,
+ nIter, classifier_name, classifier_class_name,
X_train, y_train,
randomState, outputFileName,
KFolds, nbCores, metrics, kwargs)
@@ -115,7 +116,7 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
stringAnalysis, \
imagesAnalysis, \
metricsScores = execute(name, classificationIndices, KFolds, nbCores,
- hyper_parameter_search, metrics, nIter, feat, CL_type,
+ hyper_parameter_search, metrics, nIter, feat, classifier_name,
clKWARGS, labels_names, X.shape,
y_train, y_train_pred, y_test, y_test_pred, t_end,
randomState, classifier, outputFileName)
@@ -130,39 +131,35 @@ def exec_monoview(directory, X, Y, name, labels_names, classificationIndices,
viewIndex = args["view_index"]
if testFoldsPreds is None:
testFoldsPreds = y_train_pred
- return monoview_utils.MonoviewResult(viewIndex, CL_type, feat, metricsScores,
+ return monoview_utils.MonoviewResult(viewIndex, classifier_name, feat, metricsScores,
full_labels_pred, clKWARGS,
y_test_multiclass_pred, testFoldsPreds)
# return viewIndex, [CL_type, feat, metricsScores, full_labels_pred, clKWARGS, y_test_multiclass_pred, testFoldsPreds]
-def initConstants(args, X, classificationIndices, labels_names, name, directory):
+def initConstants(args, X, classificationIndices, labels_names,
+ name, directory, view_name):
try:
kwargs = args["args"]
except KeyError:
kwargs = args
t_start = time.time()
- if type(X.attrs["name"]) == bytes:
- feat = X.attrs["name"].decode("utf-8")
- else:
- feat = X.attrs["name"]
CL_type = kwargs["classifier_name"]
- X = get_value(X)
learningRate = float(len(classificationIndices[0])) / (
len(classificationIndices[0]) + len(classificationIndices[1]))
labelsString = "-".join(labels_names)
CL_type_string = CL_type
timestr = time.strftime("%Y_%m_%d-%H_%M_%S")
- outputFileName = directory + CL_type_string + "/" + feat + "/" + timestr + "-results-" + CL_type_string + "-" + labelsString + \
+ outputFileName = directory + CL_type_string + "/" + view_name + "/" + timestr + "-results-" + CL_type_string + "-" + labelsString + \
'-learnRate_{0:.2f}'.format(
- learningRate) + '-' + name + "-" + feat + "-"
+ learningRate) + '-' + name + "-" + view_name + "-"
if not os.path.exists(os.path.dirname(outputFileName)):
try:
os.makedirs(os.path.dirname(outputFileName))
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
- return kwargs, t_start, feat, CL_type, X, learningRate, labelsString, outputFileName
+ return kwargs, t_start, view_name, CL_type, X, learningRate, labelsString, outputFileName
def init_train_test(X, Y, classificationIndices):
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview/exec_multiview.py b/multiview_platform/mono_multi_view_classifiers/multiview/exec_multiview.py
index 08288d55a6eb2e29faa0d2273c23e94636d21623..9d22a6d457487b43bd3457c72802a1530fca340c 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview/exec_multiview.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview/exec_multiview.py
@@ -33,12 +33,12 @@ def init_constants(kwargs, classification_indices, metrics, name, nb_cores, k_fo
logging.info("Info\t: Classification - Database : " + str(
name) + " ; Views : " + ", ".join(views) +
" ; Algorithm : " + classifier_name + " ; Cores : " + str(
- nbCores) + ", Train ratio : " + str(learning_rate) +
+ nb_cores) + ", Train ratio : " + str(learning_rate) +
", CV on " + str(k_folds.n_splits) + " folds")
for view_index, view_name in zip(views_indices, views):
logging.info("Info:\t Shape of " + str(view_name) + " :" + str(
- get_shape(dataset_var, view_index)))
+ dataset_var.get_shape()))
return classifier_name, t_start, views_indices, classifier_config, views, learning_rate
@@ -168,7 +168,7 @@ def exec_multiview(directory, dataset_var, name, classification_indices, k_folds
string_analysis, images_analysis, metrics_scores = analyze_results.execute(
classifier, train_labels,
test_labels, dataset_var,
- classifier_config, classificationIndices,
+ classifier_config, classification_indices,
labels_dictionary, views, nb_cores, times,
name, k_folds,
hyper_param_search, n_iter, metrics,
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py b/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
index 3370cd9938477baac14c645812a26d56045c9ae1..6e017ba764c0eb697f49ffb1bb89011e2b1976b9 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview/multiview_utils.py
@@ -87,9 +87,9 @@ class BaseMultiviewClassifier(BaseEstimator, ClassifierMixin):
def get_examples_views_indices(dataset, examples_indices, view_indices, ):
"""This function is used to get all the examples indices and view indices if needed"""
if view_indices is None:
- view_indices = np.arange(dataset["Metadata"].attrs["nbView"])
+ view_indices = np.arange(dataset.nb_view)
if examples_indices is None:
- examples_indices = range(dataset["Metadata"].attrs["datasetLength"])
+ examples_indices = range(dataset.get_nb_examples())
return examples_indices, view_indices
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/fusion_utils.py b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/fusion_utils.py
index 80f77ec9cdbc6e950d01bd241c6deb8bb79bdf4d..47a5f15f6eacf3802afdc56af8429d8d38717de4 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/fusion_utils.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/fusion_utils.py
@@ -8,7 +8,7 @@ class BaseLateFusionClassifier():
def init_monoview_estimator(self, classifier_name, classifier_index=None):
if classifier_index is not None:
- classifier_configs = self.classifier_configs[classifier_index]
+ classifier_configs = self.classifier_configs[classifier_name]
else:
classifier_configs = self.classifier_configs
if classifier_configs is not None and classifier_name in classifier_configs:
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/late_fusion_utils.py b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/late_fusion_utils.py
index c05d319c65c718381532a7e1fa85bf2b881e9ce7..189e16804eb90545cbbc1964859d4559415086c6 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/late_fusion_utils.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/late_fusion_utils.py
@@ -47,11 +47,13 @@ class LateFusionClassifier(BaseMultiviewClassifier, BaseLateFusionClassifier):
classifier_configs=None, nb_cores=1, nb_view=None, weights=None):
super(LateFusionClassifier, self).__init__(random_state)
self.verif_clf_views(classifier_names, nb_view)
+ print(classifier_names)
self.nb_view = len(classifier_names)
self.classifiers_names = classifier_names
self.classifier_configs = classifier_configs
self.monoview_estimators = [self.init_monoview_estimator(classifier_name, classifier_index)
- for classifier_index, classifier_name in self.classifiers_names]
+ for classifier_index, classifier_name
+ in enumerate(self.classifiers_names)]
self.nb_cores = nb_cores
self.accuracies = np.zeros(len(classifier_names))
self.needProbas = False
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/utils.py b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/utils.py
index 6d3c8a7b493dfdf2943e92e594e6f196c51449df..8158ec7ff13339a48eab1faa7caa28afb6f9f262 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/utils.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/additions/utils.py
@@ -69,9 +69,9 @@ class BaseMultiviewClassifier(BaseEstimator, ClassifierMixin):
def get_train_views_indices(dataset, train_indices, view_indices,):
"""This function is used to get all the examples indices and view indices if needed"""
if view_indices is None:
- view_indices = np.arange(dataset["Metadata"].attrs["nbView"])
+ view_indices = np.arange(dataset.nb_view)
if train_indices is None:
- train_indices = range(dataset["Metadata"].attrs["datasetLength"])
+ train_indices = range(dataset.get_nb_examples())
return train_indices, view_indices
diff --git a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/weighted_linear_early_fusion.py b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/weighted_linear_early_fusion.py
index 25bd9c3e95dd2fadd5f556c55bb1423aeb630aa4..aa86a71bab433060d0f8c010048058192b332510 100644
--- a/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/weighted_linear_early_fusion.py
+++ b/multiview_platform/mono_multi_view_classifiers/multiview_classifiers/weighted_linear_early_fusion.py
@@ -1,7 +1,7 @@
import numpy as np
import inspect
-from ..utils.dataset import get_v
+# from ..utils.dataset import get_v
from ..multiview.multiview_utils import BaseMultiviewClassifier, get_examples_views_indices, ConfigGenerator, get_available_monoview_classifiers
@@ -79,7 +79,7 @@ class WeightedLinearEarlyFusion(BaseMultiviewClassifier):
def hdf5_to_monoview(self, dataset, exmaples):
"""Here, we concatenate the views for the asked examples """
monoview_data = np.concatenate(
- [get_v(dataset, view_idx, exmaples)
+ [dataset.get_v(view_idx, exmaples)
for view_weight, (index, view_idx)
in zip(self.view_weights, enumerate(self.view_indices))]
, axis=1)
diff --git a/multiview_platform/mono_multi_view_classifiers/utils/dataset.py b/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
index 402f7263354f21c63df59ff3f173c0e94944ca42..2ab4003f543607ffe69be5c8897e9864c54b2534 100644
--- a/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
+++ b/multiview_platform/mono_multi_view_classifiers/utils/dataset.py
@@ -13,11 +13,14 @@ from scipy import sparse
class Dataset():
+ # The following methods use hdf5
+
def __init__(self, views=None, labels=None, are_sparse=False,
file_name="dataset.hdf5", view_names=None, path="",
hdf5_file=None, labels_names=None):
if hdf5_file is not None:
self.dataset=hdf5_file
+ self.init_attrs()
else:
if not os.path.exists(os.path.dirname(os.path.join(path, file_name))):
try:
@@ -50,27 +53,39 @@ class Dataset():
meta_data_grp.attrs["nbClass"] = len(np.unique(labels))
meta_data_grp.attrs["datasetLength"] = len(labels)
dataset_file.close()
- dataset_file = h5py.File(os.path.join(path, file_name), "r")
- self.dataset = dataset_file
+ self.update_hdf5_dataset(os.path.join(path, file_name))
+
+ def get_view_name(self, view_idx):
+ return self.dataset["View"+str(view_idx)].attrs["name"]
+
+ def init_attrs(self):
self.nb_view = self.dataset.get("Metadata").attrs["nbView"]
self.view_dict = self.get_view_dict()
+ def get_nb_examples(self):
+ return self.dataset.get("Metadata").attrs["datasetLength"]
+
def get_view_dict(self):
view_dict = {}
for view_index in range(self.nb_view):
view_dict[self.dataset.get("View" + str(view_index)).attrs["name"]] = view_index
return view_dict
- def get_label_names(self, decode=True):
+ def get_label_names(self, decode=True, example_indices=None):
+ example_indices = self.init_example_indces(example_indices)
+ selected_labels = self.get_labels(example_indices)
if decode:
return [label_name.decode("utf-8")
- for label_name in self.dataset.get("Labels").attrs["names"]]
+ for label, label_name in enumerate(self.dataset.get("Labels").attrs["names"])
+ if label in selected_labels]
else:
- return self.dataset.get("Labels").attrs["names"]
+ return [label_name
+ for label, label_name in enumerate(self.dataset.get("Labels").attrs["names"])
+ if label in selected_labels]
def init_example_indces(self, example_indices=None):
if example_indices is None:
- return range(self.dataset.get("Metadata").attrs["datasetLength"])
+ return range(self.get_nb_examples())
else:
return example_indices
@@ -97,15 +112,8 @@ class Dataset():
return sparse_mat
- # def copy(self, examples_indices, views_indices, target_dataset):
- # new_dataset = Dataset(views=,
- # labels=,
- # are_sparse=,
- # file_name=,
- # view_names=,
- # path=,
- # labels_names=)
- # return self.dataset.copy(part_name, target_dataset)
+ def get_shape(self, example_indices=None):
+ return self.get_v(0,example_indices=example_indices).shape
def get_nb_class(self, example_indices=None):
example_indices = self.init_example_indces(example_indices)
@@ -113,17 +121,150 @@ class Dataset():
def get_labels(self, example_indices=None):
example_indices = self.init_example_indces(example_indices)
- return self.dataset.get("Labels").value([example_indices])
+ return self.dataset.get("Labels").value[example_indices]
def copy_view(self, target_dataset=None, source_view_name=None,
- target_view_name=None, example_indices=None):
+ target_view_index=None, example_indices=None):
example_indices = self.init_example_indces(example_indices)
- new_d_set = target_dataset.create_dataset(target_view_name,
+ new_d_set = target_dataset.create_dataset("View"+str(target_view_index),
data=self.get_v(self.view_dict[source_view_name],
example_indices=example_indices))
- for key, value in self.get_v(self.view_dict[source_view_name]).attrs.items():
+ for key, value in self.dataset.get("View"+str(self.view_dict[source_view_name])).attrs.items():
new_d_set.attrs[key] = value
+ def init_view_names(self, view_names=None):
+ if view_names is None:
+ return [key for key in self.get_view_dict().keys()]
+ else:
+ return view_names
+
+ def update_hdf5_dataset(self, path):
+ if hasattr(self, 'dataset'):
+ self.dataset.close()
+ self.dataset = h5py.File(path, 'r')
+ self.init_attrs()
+
+ def filter(self, labels, label_names, example_indices, view_names, path):
+ dataset_file_path = os.path.join(path,self.get_name()+"_temp_filter.hdf5")
+ new_dataset_file = h5py.File(dataset_file_path,"w")
+ self.dataset.copy("Metadata", new_dataset_file)
+ new_dataset_file.get("Metadata").attrs["datasetLength"] = len(example_indices)
+ new_dataset_file.get("Metadata").attrs["nbClass"] = np.unique(labels)
+ new_dataset_file.create_dataset("Labels", data=labels)
+ new_dataset_file.get("Labels").attrs["names"] = [label_name.encode()
+ if not isinstance(label_name, bytes)
+ else label_name
+ for label_name in label_names]
+ view_names = self.init_view_names(view_names)
+ new_dataset_file.get("Metadata").attrs["nbView"] = len(view_names)
+ for new_index, view_name in enumerate(view_names):
+ self.copy_view(target_dataset=new_dataset_file,
+ source_view_name=view_name,
+ target_view_index=new_index,
+ example_indices=example_indices)
+ new_dataset_file.close()
+ self.update_hdf5_dataset(dataset_file_path)
+
+ def add_gaussian_noise(self, random_state, path,
+ noise_std=0.15):
+ """In this function, we add a guaussian noise centered in 0 with specified
+ std to each view, according to it's range (the noise will be
+ mutliplied by this range) and we crop the noisy signal according to the
+ view's attributes limits.
+ This is done by creating a new dataset, to keep clean data."""
+ noisy_dataset = h5py.File(path + self.get_name() + "_noised.hdf5", "w")
+ self.dataset.copy("Metadata", noisy_dataset)
+ self.dataset.copy("Labels", noisy_dataset)
+ for view_index in range(self.nb_view):
+ self.copy_view(target_dataset=noisy_dataset,
+ source_view_name=self.get_view_name(view_index),
+ target_view_index=view_index)
+ for view_index in range(noisy_dataset.get("Metadata").attrs["nbView"]):
+ view_key = "View" + str(view_index)
+ view_dset = noisy_dataset.get(view_key)
+ try:
+ view_limits = self.dataset[
+ "Metadata/View" + str(view_index) + "_limits"].value
+ except:
+ import pdb;pdb.set_trace()
+ view_ranges = view_limits[:, 1] - view_limits[:, 0]
+ normal_dist = random_state.normal(0, noise_std, view_dset.value.shape)
+ noise = normal_dist * view_ranges
+ noised_data = view_dset.value + noise
+ noised_data = np.where(noised_data < view_limits[:, 0],
+ view_limits[:, 0], noised_data)
+ noised_data = np.where(noised_data > view_limits[:, 1],
+ view_limits[:, 1], noised_data)
+ noisy_dataset[view_key][...] = noised_data
+ noisy_dataset_path = noisy_dataset.filename
+ noisy_dataset.close()
+ self.update_hdf5_dataset(noisy_dataset_path)
+
+
+ # The following methods are hdf5 free
+
+ def select_views_and_labels(self, nb_labels=None,
+ selected_label_names=None, random_state=None,
+ view_names = None, path_for_new="../data/"):
+ if view_names is None and selected_label_names is None and nb_labels is None:
+ pass
+ else:
+ selected_label_names = self.check_selected_label_names(nb_labels,
+ selected_label_names,
+ random_state)
+ labels, label_names, example_indices = self.select_labels(selected_label_names)
+ self.filter(labels, label_names, example_indices, view_names, path_for_new)
+ labels_dictionary = dict(
+ (labelIndex, labelName) for labelIndex, labelName in
+ enumerate(self.get_label_names()))
+ return labels_dictionary
+
+ def get_name(self):
+ """Ony works if there are not multiple dots in the files name"""
+ return self.dataset.filename.split('/')[-1].split('.')[0]
+
+ def select_labels(self, selected_label_names):
+ selected_labels = [self.get_label_names().index(label_name.decode())
+ if isinstance(label_name, bytes)
+ else self.get_label_names().index(label_name)
+ for label_name in selected_label_names]
+ selected_indices = np.array([index
+ for index, label in enumerate(self.get_labels())
+ if label in selected_labels])
+ labels = np.array([selected_labels.index(self.get_labels()[idx])
+ for idx in selected_indices])
+ return labels, selected_label_names, selected_indices
+
+ def check_selected_label_names(self, nb_labels=None,
+ selected_label_names=None,
+ random_state=np.random.RandomState(42)):
+ if selected_label_names is None or nb_labels is None or len(selected_label_names) < nb_labels:
+ if selected_label_names is None:
+ nb_labels_to_add = nb_labels
+ selected_label_names = []
+ elif nb_labels is not None:
+ nb_labels_to_add = nb_labels - len(selected_label_names)
+ else:
+ nb_labels_to_add=0
+ labels_names_to_choose = [available_label_name
+ for available_label_name
+ in self.get_label_names()
+ if available_label_name
+ not in selected_label_names]
+ added_labels_names = random_state.choice(labels_names_to_choose,
+ nb_labels_to_add,
+ replace=False)
+ selected_label_names = list(selected_label_names) + list(
+ added_labels_names)
+ elif len(selected_label_names) > nb_labels:
+ selected_label_names = list(
+ random_state.choice(selected_label_names, nb_labels,
+ replace=False))
+
+ return selected_label_names
+
+
+
def datasets_already_exist(pathF, name, nbCores):
@@ -135,30 +276,30 @@ def datasets_already_exist(pathF, name, nbCores):
pathF + name + str(coreIndex) + ".hdf5")
return allDatasetExist
-def get_v(dataset, view_index, used_indices=None):
- """Used to extract a view as a numpy array or a sparse mat from the HDF5 dataset"""
- if used_indices is None:
- used_indices = range(dataset.get("Metadata").attrs["datasetLength"])
- if type(used_indices) is int:
- return dataset.get("View" + str(view_index))[used_indices, :]
- else:
- used_indices = np.array(used_indices)
- sorted_indices = np.argsort(used_indices)
- used_indices = used_indices[sorted_indices]
-
- if not dataset.get("View" + str(view_index)).attrs["sparse"]:
- return dataset.get("View" + str(view_index))[used_indices, :][
- np.argsort(sorted_indices), :]
- else:
- sparse_mat = sparse.csr_matrix(
- (dataset.get("View" + str(view_index)).get("data").value,
- dataset.get("View" + str(view_index)).get("indices").value,
- dataset.get("View" + str(view_index)).get("indptr").value),
- shape=dataset.get("View" + str(view_index)).attrs["shape"])[
- used_indices, :][
- np.argsort(sorted_indices), :]
-
- return sparse_mat
+# def get_v(dataset, view_index, used_indices=None):
+# # """Used to extract a view as a numpy array or a sparse mat from the HDF5 dataset"""
+# # if used_indices is None:
+# # used_indices = range(dataset.get("Metadata").attrs["datasetLength"])
+# # if type(used_indices) is int:
+# # return dataset.get("View" + str(view_index))[used_indices, :]
+# # else:
+# # used_indices = np.array(used_indices)
+# # sorted_indices = np.argsort(used_indices)
+# # used_indices = used_indices[sorted_indices]
+# #
+# # if not dataset.get("View" + str(view_index)).attrs["sparse"]:
+# # return dataset.get("View" + str(view_index))[used_indices, :][
+# # np.argsort(sorted_indices), :]
+# # else:
+# # sparse_mat = sparse.csr_matrix(
+# # (dataset.get("View" + str(view_index)).get("data").value,
+# # dataset.get("View" + str(view_index)).get("indices").value,
+# # dataset.get("View" + str(view_index)).get("indptr").value),
+# # shape=dataset.get("View" + str(view_index)).attrs["shape"])[
+# # used_indices, :][
+# # np.argsort(sorted_indices), :]
+# #
+# # return sparse_mat
def get_shape(dataset, view_index):
diff --git a/multiview_platform/mono_multi_view_classifiers/utils/execution.py b/multiview_platform/mono_multi_view_classifiers/utils/execution.py
index b965f8106ea1b46be8ea740af7e0bb59a2c2df47..7bc2d155607928dd4df9eba428ab4cdf23c60cdc 100644
--- a/multiview_platform/mono_multi_view_classifiers/utils/execution.py
+++ b/multiview_platform/mono_multi_view_classifiers/utils/execution.py
@@ -115,7 +115,8 @@ def get_database_function(name, type_var):
return get_database
-def init_log_file(name, views, cl_type, log, debug, label, result_directory, add_noise, noise_std):
+def init_log_file(name, views, cl_type, log, debug, label,
+ result_directory, add_noise, noise_std):
r"""Used to init the directory where the preds will be stored and the log file.
First this function will check if the result directory already exists (only one per minute is allowed).
@@ -147,6 +148,8 @@ def init_log_file(name, views, cl_type, log, debug, label, result_directory, add
results_directory : string
Reference to the main results directory for the benchmark.
"""
+ if views is None:
+ views=[]
noise_string = "/n_"+str(int(noise_std*100))
if debug:
result_directory = result_directory + name + noise_string + \
@@ -256,31 +259,27 @@ def init_views(dataset_var, arg_views):
all_views : list of strings
Names of all the available views in the dataset.
"""
- nb_view = dataset_var.get("Metadata").attrs["nbView"]
- if arg_views != ["all"]:
+ nb_view = dataset_var.nb_view
+ if arg_views is not None:
allowed_views = arg_views
- all_views = [str(dataset_var.get("View" + str(view_index)).attrs["name"])
- if type(
- dataset_var.get("View" + str(view_index)).attrs["name"]) != bytes
- else dataset_var.get("View" + str(view_index)).attrs[
- "name"].decode("utf-8")
+ all_views = [str(dataset_var.get_view_name(view_index))
+ if type(dataset_var.get_view_name(view_index)) != bytes
+ else dataset_var.get_view_name(view_index).decode("utf-8")
for view_index in range(nb_view)]
views = []
views_indices = []
for view_index in range(nb_view):
- view_name = dataset_var.get("View" + str(view_index)).attrs["name"]
+ view_name = dataset_var.get_view_name(view_index)
if type(view_name) == bytes:
view_name = view_name.decode("utf-8")
if view_name in allowed_views:
views.append(view_name)
views_indices.append(view_index)
else:
- views = [str(dataset_var.get("View" + str(viewIndex)).attrs["name"])
- if type(
- dataset_var.get("View" + str(viewIndex)).attrs["name"]) != bytes
- else dataset_var.get("View" + str(viewIndex)).attrs["name"].decode(
- "utf-8")
- for viewIndex in range(nb_view)]
+ views = [str(dataset_var.get_view_name(view_index))
+ if type(dataset_var.get_view_name(view_index)) != bytes
+ else dataset_var.get_view_name(view_index).decode("utf-8")
+ for view_index in range(nb_view)]
views_indices = range(nb_view)
all_views = views
return views, views_indices, all_views
diff --git a/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py b/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
index d968500a489aa74fe712489d54eace498985a0f0..c4c5d376cb7960e244413b1f1e76ca34b75a5c19 100644
--- a/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
+++ b/multiview_platform/mono_multi_view_classifiers/utils/get_multiview_db.py
@@ -125,143 +125,144 @@ class DatasetError(Exception):
def __init__(self, *args, **kwargs):
Exception.__init__(self, *args, **kwargs)
-
-def get_classes(labels):
- labels_set = set(list(labels))
- nb_labels = len(labels_set)
- if nb_labels >= 2:
- return labels_set
- else:
- raise DatasetError("Dataset must have at least two different labels")
-
-
-def all_asked_labels_are_available(asked_labels_names_set,
- available_labels_names):
- for askedLabelName in asked_labels_names_set:
- if askedLabelName in available_labels_names:
- pass
- else:
- return False
- return True
-
-
-def fill_label_names(nb_class, asked_labels_names, random_state,
- available_labels_names):
- if len(asked_labels_names) < nb_class:
- nb_labels_to_add = nb_class - len(asked_labels_names)
- labels_names_to_choose = [available_label_name
- for available_label_name
- in available_labels_names
- if available_label_name
- not in asked_labels_names]
- added_labels_names = random_state.choice(labels_names_to_choose,
- nb_labels_to_add, replace=False)
- asked_labels_names = list(asked_labels_names) + list(added_labels_names)
- asked_labels_names_set = set(asked_labels_names)
-
- elif len(asked_labels_names) > nb_class:
- asked_labels_names = list(
- random_state.choice(asked_labels_names, nb_class, replace=False))
- asked_labels_names_set = set(asked_labels_names)
-
- else:
- asked_labels_names_set = set(asked_labels_names)
-
- return asked_labels_names, asked_labels_names_set
-
-
-def get_all_labels(full_labels, available_labels_names):
- new_labels = full_labels
- new_labels_names = available_labels_names
- used_indices = np.arange(len(full_labels))
- return new_labels, new_labels_names, used_indices
-
-
-def select_asked_labels(asked_labels_names_set, available_labels_names,
- asked_labels_names, full_labels):
- if all_asked_labels_are_available(asked_labels_names_set, available_labels_names):
- used_labels = [available_labels_names.index(asked_label_name) for
- asked_label_name in asked_labels_names]
- used_indices = np.array(
- [labelIndex for labelIndex, label in enumerate(full_labels) if
- label in used_labels])
- new_labels = np.array([used_labels.index(label) for label in full_labels if
- label in used_labels])
- new_labels_names = [available_labels_names[usedLabel] for usedLabel in
- used_labels]
- return new_labels, new_labels_names, used_indices
- else:
- raise DatasetError("Asked labels are not all available in the dataset")
-
-
-def filter_labels(labels_set, asked_labels_names_set, full_labels,
- available_labels_names, asked_labels_names):
- if len(labels_set) > 2:
- if asked_labels_names == available_labels_names:
- new_labels, new_labels_names, used_indices = \
- get_all_labels(full_labels, available_labels_names)
- elif len(asked_labels_names_set) <= len(labels_set):
- new_labels, new_labels_names, used_indices = select_asked_labels(
- asked_labels_names_set, available_labels_names,
- asked_labels_names, full_labels)
- else:
- raise DatasetError(
- "Asked more labels than available in the dataset. Available labels are : " +
- ", ".join(available_labels_names))
-
- else:
- new_labels, new_labels_names, used_indices = get_all_labels(full_labels,
- available_labels_names)
- return new_labels, new_labels_names, used_indices
-
-
-def filter_views(dataset_file, temp_dataset, views, used_indices):
- new_view_index = 0
- if views == [""]:
- for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
- copyhdf5_dataset(dataset_file, temp_dataset, "View" + str(view_index),
- "View" + str(view_index), used_indices)
- else:
- for asked_view_name in views:
- for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
- view_name = dataset_file.get("View" + str(view_index)).attrs["name"]
- if type(view_name) == bytes:
- view_name = view_name.decode("utf-8")
- if view_name == asked_view_name:
- copyhdf5_dataset(dataset_file, temp_dataset,
- "View" + str(view_index),
- "View" + str(new_view_index), used_indices)
- new_view_name = \
- temp_dataset.get("View" + str(new_view_index)).attrs["name"]
- if type(new_view_name) == bytes:
- temp_dataset.get("View" + str(new_view_index)).attrs[
- "name"] = new_view_name.decode("utf-8")
-
- new_view_index += 1
- else:
- pass
- temp_dataset.get("Metadata").attrs["nbView"] = len(views)
-
-
-def copyhdf5_dataset(source_data_file, destination_data_file, source_dataset_name,
- destination_dataset_name, used_indices):
- """Used to copy a view in a new dataset file using only the examples of
- usedIndices, and copying the args"""
- new_d_set = destination_data_file.create_dataset(destination_dataset_name,
- data=source_data_file.get(
- source_dataset_name).value[
- used_indices, :])
- if "sparse" in source_data_file.get(source_dataset_name).attrs.keys() and \
- source_data_file.get(source_dataset_name).attrs["sparse"]:
- # TODO : Support sparse
- pass
- else:
- for key, value in source_data_file.get(source_dataset_name).attrs.items():
- new_d_set.attrs[key] = value
+#
+# def get_classes(labels):
+# labels_set = set(list(labels))
+# nb_labels = len(labels_set)
+# if nb_labels >= 2:
+# return labels_set
+# else:
+# raise DatasetError("Dataset must have at least two different labels")
+#
+#
+# def all_asked_labels_are_available(asked_labels_names_set,
+# available_labels_names):
+# for askedLabelName in asked_labels_names_set:
+# if askedLabelName in available_labels_names:
+# pass
+# else:
+# return False
+# return True
+#
+#
+# def fill_label_names(nb_labels, selected_label_names, random_state,
+# available_labels_names):
+# if len(selected_label_names) < nb_labels:
+# nb_labels_to_add = nb_labels - len(selected_label_names)
+# labels_names_to_choose = [available_label_name
+# for available_label_name
+# in available_labels_names
+# if available_label_name
+# not in selected_label_names]
+# added_labels_names = random_state.choice(labels_names_to_choose,
+# nb_labels_to_add, replace=False)
+# selected_label_names = list(selected_label_names) + list(added_labels_names)
+# asked_labels_names_set = set(selected_label_names)
+#
+# elif len(selected_label_names) > nb_labels:
+# selected_label_names = list(
+# random_state.choice(selected_label_names, nb_labels, replace=False))
+# asked_labels_names_set = set(selected_label_names)
+#
+# else:
+# asked_labels_names_set = set(selected_label_names)
+#
+# return selected_label_names, asked_labels_names_set
+#
+#
+# def get_all_labels(full_labels, available_labels_names):
+# new_labels = full_labels
+# new_labels_names = available_labels_names
+# used_indices = np.arange(len(full_labels))
+# return new_labels, new_labels_names, used_indices
+#
+#
+# def select_asked_labels(asked_labels_names_set, available_labels_names,
+# asked_labels_names, full_labels):
+# if all_asked_labels_are_available(asked_labels_names_set, available_labels_names):
+# used_labels = [available_labels_names.index(asked_label_name) for
+# asked_label_name in asked_labels_names]
+# used_indices = np.array(
+# [labelIndex for labelIndex, label in enumerate(full_labels) if
+# label in used_labels])
+# new_labels = np.array([used_labels.index(label) for label in full_labels if
+# label in used_labels])
+# new_labels_names = [available_labels_names[usedLabel] for usedLabel in
+# used_labels]
+# return new_labels, new_labels_names, used_indices
+# else:
+# raise DatasetError("Asked labels are not all available in the dataset")
+#
+#
+# def filter_labels(labels_set, asked_labels_names_set, full_labels,
+# available_labels_names, asked_labels_names):
+# if len(labels_set) > 2:
+# if asked_labels_names == available_labels_names:
+# new_labels, new_labels_names, used_indices = \
+# get_all_labels(full_labels, available_labels_names)
+# elif len(asked_labels_names_set) <= len(labels_set):
+# new_labels, new_labels_names, used_indices = select_asked_labels(
+# asked_labels_names_set, available_labels_names,
+# asked_labels_names, full_labels)
+# else:
+# raise DatasetError(
+# "Asked more labels than available in the dataset. Available labels are : " +
+# ", ".join(available_labels_names))
+#
+# else:
+# new_labels, new_labels_names, used_indices = get_all_labels(full_labels,
+# available_labels_names)
+# return new_labels, new_labels_names, used_indices
+#
+#
+# def filter_views(dataset_file, temp_dataset, views, used_indices):
+# new_view_index = 0
+# if views == [""]:
+# for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
+# copyhdf5_dataset(dataset_file, temp_dataset, "View" + str(view_index),
+# "View" + str(view_index), used_indices)
+# else:
+# for asked_view_name in views:
+# for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
+# view_name = dataset_file.get("View" + str(view_index)).attrs["name"]
+# if type(view_name) == bytes:
+# view_name = view_name.decode("utf-8")
+# if view_name == asked_view_name:
+# copyhdf5_dataset(dataset_file, temp_dataset,
+# "View" + str(view_index),
+# "View" + str(new_view_index), used_indices)
+# new_view_name = \
+# temp_dataset.get("View" + str(new_view_index)).attrs["name"]
+# if type(new_view_name) == bytes:
+# temp_dataset.get("View" + str(new_view_index)).attrs[
+# "name"] = new_view_name.decode("utf-8")
+#
+# new_view_index += 1
+# else:
+# pass
+# temp_dataset.get("Metadata").attrs["nbView"] = len(views)
+#
+#
+# def copyhdf5_dataset(source_data_file, destination_data_file, source_dataset_name,
+# destination_dataset_name, used_indices):
+# """Used to copy a view in a new dataset file using only the examples of
+# usedIndices, and copying the args"""
+# new_d_set = destination_data_file.create_dataset(destination_dataset_name,
+# data=source_data_file.get(
+# source_dataset_name).value[
+# used_indices, :])
+# if "sparse" in source_data_file.get(source_dataset_name).attrs.keys() and \
+# source_data_file.get(source_dataset_name).attrs["sparse"]:
+# # TODO : Support sparse
+# pass
+# else:
+# for key, value in source_data_file.get(source_dataset_name).attrs.items():
+# new_d_set.attrs[key] = value
def get_classic_db_hdf5(views, path_f, name_DB, nb_class, asked_labels_names,
- random_state, full=False, add_noise=False, noise_std=0.15,):
+ random_state, full=False, add_noise=False, noise_std=0.15,
+ path_for_new="../data/"):
"""Used to load a hdf5 database"""
if full:
dataset_file = h5py.File(path_f + name_DB + ".hdf5", "r")
@@ -269,88 +270,62 @@ def get_classic_db_hdf5(views, path_f, name_DB, nb_class, asked_labels_names,
dataset_name = name_DB
labels_dictionary = dict((label_index, label_name)
for label_index, label_name
- in dataset.get_label_names())
+ in enumerate(dataset.get_label_names()))
else:
- asked_labels_names = [asked_label_name.encode("utf8") for asked_label_name in
- asked_labels_names]
- base_dataset_file = h5py.File(path_f + name_DB + ".hdf5", "r")
- full_labels = base_dataset_file.get("Labels").value
- dataset_file = h5py.File(path_f + name_DB + "_temp_view_label_select.hdf5",
- "w")
- dataset_name = name_DB + "_temp_view_label_select"
- base_dataset_file.copy("Metadata", dataset_file)
- labels_set = get_classes(full_labels)
- available_labels_names = list(
- base_dataset_file.get("Labels").attrs["names"])
- asked_labels_names, asked_labels_names_set = fill_label_names(nb_class,
- asked_labels_names,
- random_state,
- available_labels_names)
-
- new_labels, new_labels_names, used_indices = filter_labels(labels_set,
- asked_labels_names_set,
- full_labels,
- available_labels_names,
- asked_labels_names)
- dataset_file.get("Metadata").attrs["datasetLength"] = len(used_indices)
- dataset_file.get("Metadata").attrs["nbClass"] = nb_class
- dataset_file.create_dataset("Labels", data=new_labels)
- dataset_file.get("Labels").attrs["names"] = new_labels_names
- filter_views(base_dataset_file, dataset_file, views, used_indices)
+ dataset_file = h5py.File(path_f + name_DB + ".hdf5", "r")
+ dataset = Dataset(hdf5_file=dataset_file)
+ labels_dictionary = dataset.select_views_and_labels(nb_labels=nb_class,
+ selected_label_names=asked_labels_names,
+ view_names=views, random_state=random_state,
+ path_for_new=path_for_new)
+ dataset_name = dataset.get_name()
- labels_dictionary = dict(
- (labelIndex, labelName.decode("utf-8")) for labelIndex, labelName in
- enumerate(dataset_file.get("Labels").attrs["names"]))
- dataset_file.close()
- dataset_file = h5py.File(path_f + name_DB + "_temp_view_label_select.hdf5",
- "r")
if add_noise:
- dataset_file, dataset_name = add_gaussian_noise(dataset_file, random_state,
- path_f, dataset_name,
- noise_std)
+ dataset.add_gaussian_noise(random_state, path_for_new, noise_std)
+ dataset_name = dataset.get_name()
else:
pass
return dataset_file, labels_dictionary, dataset_name
-
-def add_gaussian_noise(dataset_file, random_state, path_f, dataset_name,
- noise_std=0.15):
- """In this function, we add a guaussian noise centered in 0 with specified
- std to each view, according to it's range (the noise will be
- mutliplied by this range) and we crop the noisy signal according to the
- view's attributes limits.
- This is done by creating a new dataset, to keep clean data."""
- noisy_dataset = h5py.File(path_f + dataset_name + "_noised.hdf5", "w")
- dataset_file.copy("Metadata", noisy_dataset)
- dataset_file.copy("Labels", noisy_dataset)
- for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
- dataset_file.copy("View" + str(view_index), noisy_dataset)
- for view_index in range(noisy_dataset.get("Metadata").attrs["nbView"]):
- view_name = "View" + str(view_index)
- view_dset = noisy_dataset.get(view_name)
- view_limits = dataset_file[
- "Metadata/View" + str(view_index) + "_limits"].value
- view_ranges = view_limits[:, 1] - view_limits[:, 0]
- normal_dist = random_state.normal(0, noise_std, view_dset.value.shape)
- noise = normal_dist * view_ranges
- noised_data = view_dset.value + noise
- noised_data = np.where(noised_data < view_limits[:, 0],
- view_limits[:, 0], noised_data)
- noised_data = np.where(noised_data > view_limits[:, 1],
- view_limits[:, 1], noised_data)
- noisy_dataset[view_name][...] = noised_data
- original_dataset_filename = dataset_file.filename
- dataset_file.close()
- noisy_dataset.close()
- noisy_dataset = h5py.File(path_f + dataset_name + "_noised.hdf5", "r")
- if "_temp_" in original_dataset_filename:
- os.remove(original_dataset_filename)
- return noisy_dataset, dataset_name + "_noised"
+#
+# def add_gaussian_noise(dataset_file, random_state, path_f, dataset_name,
+# noise_std=0.15):
+# """In this function, we add a guaussian noise centered in 0 with specified
+# std to each view, according to it's range (the noise will be
+# mutliplied by this range) and we crop the noisy signal according to the
+# view's attributes limits.
+# This is done by creating a new dataset, to keep clean data."""
+# noisy_dataset = h5py.File(path_f + dataset_name + "_noised.hdf5", "w")
+# dataset_file.copy("Metadata", noisy_dataset)
+# dataset_file.copy("Labels", noisy_dataset)
+# for view_index in range(dataset_file.get("Metadata").attrs["nbView"]):
+# dataset_file.copy("View" + str(view_index), noisy_dataset)
+# for view_index in range(noisy_dataset.get("Metadata").attrs["nbView"]):
+# view_name = "View" + str(view_index)
+# view_dset = noisy_dataset.get(view_name)
+# view_limits = dataset_file[
+# "Metadata/View" + str(view_index) + "_limits"].value
+# view_ranges = view_limits[:, 1] - view_limits[:, 0]
+# normal_dist = random_state.normal(0, noise_std, view_dset.value.shape)
+# noise = normal_dist * view_ranges
+# noised_data = view_dset.value + noise
+# noised_data = np.where(noised_data < view_limits[:, 0],
+# view_limits[:, 0], noised_data)
+# noised_data = np.where(noised_data > view_limits[:, 1],
+# view_limits[:, 1], noised_data)
+# noisy_dataset[view_name][...] = noised_data
+# original_dataset_filename = dataset_file.filename
+# dataset_file.close()
+# noisy_dataset.close()
+# noisy_dataset = h5py.File(path_f + dataset_name + "_noised.hdf5", "r")
+# if "_temp_" in original_dataset_filename:
+# os.remove(original_dataset_filename)
+# return noisy_dataset, dataset_name + "_noised"
def get_classic_db_csv(views, pathF, nameDB, NB_CLASS, askedLabelsNames,
random_state, full=False, add_noise=False, noise_std=0.15,
- delimiter=","):
+ delimiter=",", path_for_new="../data/"):
# TODO : Update this one
labels_names = np.genfromtxt(pathF + nameDB + "-labels-names.csv",
dtype='str', delimiter=delimiter)
@@ -380,7 +355,8 @@ def get_classic_db_csv(views, pathF, nameDB, NB_CLASS, askedLabelsNames,
datasetFile.close()
datasetFile, labelsDictionary, dataset_name = get_classic_db_hdf5(views, pathF, nameDB,
NB_CLASS, askedLabelsNames,
- random_state, full)
+ random_state, full,
+ path_for_new=path_for_new)
return datasetFile, labelsDictionary, dataset_name
diff --git a/multiview_platform/tests/test_ExecClassif.py b/multiview_platform/tests/test_ExecClassif.py
index 6c7dd9719a06f5f1d43dc6e3d343c66006b77f3c..5348634a6ef1760ab04df2b71497d84ca1f488c3 100644
--- a/multiview_platform/tests/test_ExecClassif.py
+++ b/multiview_platform/tests/test_ExecClassif.py
@@ -4,7 +4,7 @@ import unittest
import h5py
import numpy as np
-from .utils import rm_tmp, tmp_path
+from .utils import rm_tmp, tmp_path, test_dataset
from multiview_platform.mono_multi_view_classifiers import exec_classif
@@ -232,11 +232,16 @@ class Test_execBenchmark(unittest.TestCase):
@classmethod
def setUpClass(cls):
rm_tmp()
+<<<<<<< HEAD
os.mkdir("./tests/tmp_tests")
cls.Dataset = h5py.File(
tmp_path+"test_file.hdf5", "w")
cls.labels = cls.Dataset.create_dataset("Labels",
data=np.array([0, 1, 2]))
+=======
+ os.mkdir(tmp_path)
+ cls.Dataset = test_dataset
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
cls.argument_dictionaries = [{"a": 4, "args": {}}]
cls.args = {
"Base":{"name": "chicken_is_heaven", "type": "type", "pathf": "pathF"},
@@ -294,12 +299,7 @@ class Test_execBenchmark(unittest.TestCase):
@classmethod
def tearDownClass(cls):
- cls.Dataset.close()
- path = tmp_path
- for file_name in os.listdir(path):
- os.remove(os.path.join(path, file_name))
- os.rmdir(path)
-
+ rm_tmp()
def fakeExecMono(directory, name, labels_names, classification_indices, k_folds,
coreIndex, type, pathF, random_state, labels,
diff --git a/multiview_platform/tests/test_database.hdf5 b/multiview_platform/tests/test_database.hdf5
new file mode 100644
index 0000000000000000000000000000000000000000..63206a1219daf9e09d9a91d5a2440ef19a88af82
Binary files /dev/null and b/multiview_platform/tests/test_database.hdf5 differ
diff --git a/multiview_platform/tests/test_mono_view/test_ExecClassifMonoView.py b/multiview_platform/tests/test_mono_view/test_ExecClassifMonoView.py
index e9f5ce5ef29ad608105921d342d04c3b8bc7ca00..469adf7d3e89e9cfdbcc6cc9f216b2338bae4d71 100644
--- a/multiview_platform/tests/test_mono_view/test_ExecClassifMonoView.py
+++ b/multiview_platform/tests/test_mono_view/test_ExecClassifMonoView.py
@@ -17,6 +17,7 @@ class Test_initConstants(unittest.TestCase):
def setUpClass(cls):
rm_tmp()
os.mkdir("multiview_platform/tests/tmp_tests")
+ cls.view_name="test_dataset"
cls.datasetFile = h5py.File(
tmp_path+"test.hdf5", "w")
cls.random_state = np.random.RandomState(42)
@@ -45,7 +46,8 @@ class Test_initConstants(unittest.TestCase):
cls.classification_indices,
cls.labels_names,
cls.name,
- cls.directory)
+ cls.directory,
+ cls.view_name)
cls.assertEqual(kwargs, cls.args)
cls.assertEqual(feat, "test_dataset")
cls.assertEqual(CL_type, "test_clf")
diff --git a/multiview_platform/tests/test_multiview_classifiers/test_weighted_linear_early_fusion.py b/multiview_platform/tests/test_multiview_classifiers/test_weighted_linear_early_fusion.py
index 3c274c2880108f61d295788b56eb2e473d392742..65541a03c067f8e2a26e5367314b1fc89d7af9ed 100644
--- a/multiview_platform/tests/test_multiview_classifiers/test_weighted_linear_early_fusion.py
+++ b/multiview_platform/tests/test_multiview_classifiers/test_weighted_linear_early_fusion.py
@@ -1,10 +1,9 @@
import unittest
import numpy as np
-import h5py
import os
-from ..utils import rm_tmp, tmp_path
+from ..utils import rm_tmp, tmp_path, test_dataset
from multiview_platform.mono_multi_view_classifiers.multiview_classifiers import \
weighted_linear_early_fusion
@@ -16,34 +15,17 @@ class Test_WeightedLinearEarlyFusion(unittest.TestCase):
rm_tmp()
cls.random_state = np.random.RandomState(42)
cls.view_weights = [0.5, 0.5]
- os.mkdir("multiview_platform/tests/tmp_tests")
- cls.dataset_file = h5py.File(
- tmp_path+"test_file.hdf5", "w")
- cls.labels = cls.dataset_file.create_dataset("Labels",
- data=np.array([0, 1, 0, 0, 1]))
- cls.view0_data = cls.random_state.randint(1,10,size=(5, 4))
- view0 = cls.dataset_file.create_dataset("View0", data=cls.view0_data)
- view0.attrs["sparse"] = False
- cls.view1_data = cls.random_state.randint(1, 10, size=(5, 4))
- view1 = cls.dataset_file.create_dataset("View1", data=cls.view1_data)
- view1.attrs["sparse"] = False
- metaDataGrp = cls.dataset_file.create_group("Metadata")
- metaDataGrp.attrs["nbView"] = 2
- metaDataGrp.attrs["nbClass"] = 2
- metaDataGrp.attrs["datasetLength"] = 5
cls.monoview_classifier_name = "decision_tree"
cls.monoview_classifier_config = {"max_depth":1, "criterion": "gini", "splitter": "best"}
cls.classifier = weighted_linear_early_fusion.WeightedLinearEarlyFusion(
random_state=cls.random_state, view_weights=cls.view_weights,
monoview_classifier_name=cls.monoview_classifier_name,
monoview_classifier_config=cls.monoview_classifier_config)
+ cls.dataset = test_dataset
@classmethod
def tearDownClass(cls):
- cls.dataset_file.close()
- for file_name in os.listdir("multiview_platform/tests/tmp_tests"):
- os.remove(os.path.join("multiview_platform/tests/tmp_tests", file_name))
- os.rmdir("multiview_platform/tests/tmp_tests")
+ rm_tmp()
def test_simple(self):
np.testing.assert_array_equal(self.view_weights, self.classifier.view_weights)
@@ -51,37 +33,35 @@ class Test_WeightedLinearEarlyFusion(unittest.TestCase):
def test_fit(self):
self.assertRaises(AttributeError, getattr,
self.classifier.monoview_classifier, "classes_")
- self.classifier.fit(self.dataset_file, self.labels, None, None)
+ self.classifier.fit(self.dataset, test_dataset.get_labels(), None, None)
np.testing.assert_array_equal(self.classifier.monoview_classifier.classes_,
np.array([0,1]))
def test_predict(self):
- self.classifier.fit(self.dataset_file, self.labels, None, None)
- predicted_labels = self.classifier.predict(self.dataset_file, None, None)
- np.testing.assert_array_equal(predicted_labels, self.labels)
+ self.classifier.fit(self.dataset, test_dataset.get_labels(), None, None)
+ predicted_labels = self.classifier.predict(self.dataset, None, None)
+ np.testing.assert_array_equal(predicted_labels, test_dataset.get_labels())
def test_transform_data_to_monoview_simple(self):
-
-
- example_indices, X = self.classifier.transform_data_to_monoview(self.dataset_file,
+ example_indices, X = self.classifier.transform_data_to_monoview(self.dataset,
None, None)
- self.assertEqual(X.shape, (5,8))
- np.testing.assert_array_equal(X, np.concatenate((self.view0_data, self.view1_data), axis=1))
+ self.assertEqual(X.shape, (5,12))
+ np.testing.assert_array_equal(X, np.concatenate((self.dataset.get_v(0), self.dataset.get_v(1)), axis=1))
np.testing.assert_array_equal(example_indices, np.arange(5))
def test_transform_data_to_monoview_view_select(self):
example_indices, X = self.classifier.transform_data_to_monoview(
- self.dataset_file,
+ self.dataset,
None, np.array([0]))
- self.assertEqual(X.shape, (5, 4))
- np.testing.assert_array_equal(X, self.view0_data)
+ self.assertEqual(X.shape, (5, 6))
+ np.testing.assert_array_equal(X, self.dataset.get_v(0))
np.testing.assert_array_equal(example_indices, np.arange(5))
- def test_transform_data_to_monoview_view_select(self):
+ def test_transform_data_to_monoview_example_view_select(self):
example_indices, X = self.classifier.transform_data_to_monoview(
- self.dataset_file,
+ self.dataset,
np.array([1,2,3]), np.array([0]))
- self.assertEqual(X.shape, (3, 4))
- np.testing.assert_array_equal(X, self.view0_data[np.array([1,2,3]), :])
+ self.assertEqual(X.shape, (3, 6))
+ np.testing.assert_array_equal(X, self.dataset.get_v(0)[np.array([1,2,3]), :])
np.testing.assert_array_equal(example_indices, np.array([1,2,3]))
diff --git a/multiview_platform/tests/test_utils/test_GetMultiviewDB.py b/multiview_platform/tests/test_utils/test_GetMultiviewDB.py
index bb84359ea450ad8d7b0329a7cab3073947826bf7..7706331636fae346ef4b2298d8a4c69c60869fe6 100644
--- a/multiview_platform/tests/test_utils/test_GetMultiviewDB.py
+++ b/multiview_platform/tests/test_utils/test_GetMultiviewDB.py
@@ -5,6 +5,7 @@ import h5py
import numpy as np
from ...mono_multi_view_classifiers.utils import get_multiview_db
+<<<<<<< HEAD
from ..utils import rm_tmp, tmp_path
@@ -97,9 +98,338 @@ class Test_filterViews(unittest.TestCase):
os.remove(tmp_path+"test_copy.hdf5")
os.remove(tmp_path+"test_copy_temp.hdf5")
os.rmdir("multiview_platform/tests/tmp_tests")
+=======
+from ..utils import rm_tmp, tmp_path, test_dataset
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
+# class Test_copyhdf5Dataset(unittest.TestCase):
#
+# @classmethod
+# def setUpClass(cls):
+# rm_tmp()
+# cls.random_state = np.random.RandomState(42)
+# if not os.path.exists("multiview_platform/tests/tmp_tests"):
+# os.mkdir("multiview_platform/tests/tmp_tests")
+# cls.dataset_file = h5py.File(
+# tmp_path+"test_copy.hdf5", "w")
+# cls.dataset = cls.dataset_file.create_dataset("test",
+# data=cls.random_state.randint(
+# 0, 100, (10, 20)))
+# cls.dataset.attrs["test_arg"] = "Am I copied"
+#
+# def test_simple_copy(cls):
+# get_multiview_db.copyhdf5_dataset(cls.dataset_file, cls.dataset_file,
+# "test", "test_copy_1", np.arange(10))
+# np.testing.assert_array_equal(cls.dataset_file.get("test").value,
+# cls.dataset_file.get("test_copy_1").value)
+# cls.assertEqual("Am I copied",
+# cls.dataset_file.get("test_copy_1").attrs["test_arg"])
+#
+# def test_copy_only_some_indices(cls):
+# usedIndices = cls.random_state.choice(10, 6, replace=False)
+# get_multiview_db.copyhdf5_dataset(cls.dataset_file, cls.dataset_file,
+# "test", "test_copy", usedIndices)
+# np.testing.assert_array_equal(
+# cls.dataset_file.get("test").value[usedIndices, :],
+# cls.dataset_file.get("test_copy").value)
+# cls.assertEqual("Am I copied",
+# cls.dataset_file.get("test_copy").attrs["test_arg"])
+#
+# @classmethod
+# def tearDownClass(cls):
+# os.remove(tmp_path+"test_copy.hdf5")
+# os.rmdir("multiview_platform/tests/tmp_tests")
+#
+#
+# class Test_filterViews(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# rm_tmp()
+# cls.random_state = np.random.RandomState(42)
+# cls.views = ["test_view_1", "test_view_2"]
+# if not os.path.exists("multiview_platform/tests/tmp_tests"):
+# os.mkdir("multiview_platform/tests/tmp_tests")
+# cls.dataset_file = h5py.File(
+# tmp_path+"test_copy.hdf5", "w")
+# cls.metadata_group = cls.dataset_file.create_group("Metadata")
+# cls.metadata_group.attrs["nbView"] = 4
+#
+# for i in range(4):
+# cls.dataset = cls.dataset_file.create_dataset("View" + str(i),
+# data=cls.random_state.randint(
+# 0, 100, (10, 20)))
+# cls.dataset.attrs["name"] = "test_view_" + str(i)
+#
+# def test_simple_filter(cls):
+# cls.temp_dataset_file = h5py.File(
+# tmp_path+"test_copy_temp.hdf5", "w")
+# cls.dataset_file.copy("Metadata", cls.temp_dataset_file)
+# get_multiview_db.filter_views(cls.dataset_file, cls.temp_dataset_file,
+# cls.views, np.arange(10))
+# cls.assertEqual(cls.dataset_file.get("View1").attrs["name"],
+# cls.temp_dataset_file.get("View0").attrs["name"])
+# np.testing.assert_array_equal(cls.dataset_file.get("View2").value,
+# cls.temp_dataset_file.get("View1").value)
+# cls.assertEqual(cls.temp_dataset_file.get("Metadata").attrs["nbView"],
+# 2)
+#
+# def test_filter_view_and_examples(cls):
+# cls.temp_dataset_file = h5py.File(
+# tmp_path+"test_copy_temp.hdf5", "w")
+# cls.dataset_file.copy("Metadata", cls.temp_dataset_file)
+# usedIndices = cls.random_state.choice(10, 6, replace=False)
+# get_multiview_db.filter_views(cls.dataset_file, cls.temp_dataset_file,
+# cls.views, usedIndices)
+# np.testing.assert_array_equal(
+# cls.dataset_file.get("View1").value[usedIndices, :],
+# cls.temp_dataset_file.get("View0").value)
+# cls.temp_dataset_file.close()
+#
+# @classmethod
+# def tearDownClass(cls):
+# os.remove(tmp_path+"test_copy.hdf5")
+# os.remove(tmp_path+"test_copy_temp.hdf5")
+# os.rmdir("multiview_platform/tests/tmp_tests")
+#
+#
+# #
+# class Test_filterLabels(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.random_state = np.random.RandomState(42)
+# cls.labelsSet = set(range(4))
+# cls.askedLabelsNamesSet = {"test_label_1", "test_label_3"}
+# cls.fullLabels = cls.random_state.randint(0, 4, 10)
+# cls.availableLabelsNames = ["test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"]
+# cls.askedLabelsNames = ["test_label_1", "test_label_3"]
+#
+# def test_simple(cls):
+# newLabels, \
+# newLabelsNames, \
+# usedIndices = get_multiview_db.filter_labels(cls.labelsSet,
+# cls.askedLabelsNamesSet,
+# cls.fullLabels,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames)
+# cls.assertEqual(["test_label_1", "test_label_3"], newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.array([1, 5, 9]))
+# np.testing.assert_array_equal(newLabels, np.array([1, 1, 0]))
+#
+# def test_biclasse(cls):
+# cls.labelsSet = {0, 1}
+# cls.fullLabels = cls.random_state.randint(0, 2, 10)
+# cls.availableLabelsNames = ["test_label_0", "test_label_1"]
+# newLabels, \
+# newLabelsNames, \
+# usedIndices = get_multiview_db.filter_labels(cls.labelsSet,
+# cls.askedLabelsNamesSet,
+# cls.fullLabels,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames)
+# cls.assertEqual(cls.availableLabelsNames, newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.arange(10))
+# np.testing.assert_array_equal(newLabels, cls.fullLabels)
+#
+# def test_asked_too_many_labels(cls):
+# cls.askedLabelsNamesSet = {"test_label_0", "test_label_1",
+# "test_label_2", "test_label_3",
+# "chicken_is_heaven"}
+# with cls.assertRaises(get_multiview_db.DatasetError) as catcher:
+# get_multiview_db.filter_labels(cls.labelsSet,
+# cls.askedLabelsNamesSet,
+# cls.fullLabels,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames)
+# exception = catcher.exception
+#
+# def test_asked_all_labels(cls):
+# cls.askedLabelsNamesSet = {"test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"}
+# cls.askedLabelsNames = ["test_label_0", "test_label_1", "test_label_2",
+# "test_label_3"]
+# newLabels, \
+# newLabelsNames, \
+# usedIndices = get_multiview_db.filter_labels(cls.labelsSet,
+# cls.askedLabelsNamesSet,
+# cls.fullLabels,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames)
+# cls.assertEqual(cls.availableLabelsNames, newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.arange(10))
+# np.testing.assert_array_equal(newLabels, cls.fullLabels)
+#
+#
+# class Test_selectAskedLabels(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.random_state = np.random.RandomState(42)
+# cls.askedLabelsNamesSet = {"test_label_1", "test_label_3"}
+# cls.fullLabels = cls.random_state.randint(0, 4, 10)
+# cls.availableLabelsNames = ["test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"]
+# cls.askedLabelsNames = ["test_label_1", "test_label_3"]
+#
+# def test_simple(cls):
+# newLabels, \
+# newLabelsNames, \
+# usedIndices = get_multiview_db.select_asked_labels(cls.askedLabelsNamesSet,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames,
+# cls.fullLabels)
+# cls.assertEqual(["test_label_1", "test_label_3"], newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.array([1, 5, 9]))
+# np.testing.assert_array_equal(newLabels, np.array([1, 1, 0]))
+#
+# def test_asked_all_labels(cls):
+# cls.askedLabelsNamesSet = {"test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"}
+# cls.askedLabelsNames = ["test_label_0", "test_label_1", "test_label_2",
+# "test_label_3"]
+# newLabels, \
+# newLabelsNames, \
+# usedIndices = get_multiview_db.select_asked_labels(cls.askedLabelsNamesSet,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames,
+# cls.fullLabels)
+# cls.assertEqual(cls.availableLabelsNames, newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.arange(10))
+# np.testing.assert_array_equal(newLabels, cls.fullLabels)
+#
+# def test_asked_unavailable_labels(cls):
+# cls.askedLabelsNamesSet = {"test_label_1", "test_label_3",
+# "chicken_is_heaven"}
+# with cls.assertRaises(get_multiview_db.DatasetError) as catcher:
+# get_multiview_db.select_asked_labels(cls.askedLabelsNamesSet,
+# cls.availableLabelsNames,
+# cls.askedLabelsNames,
+# cls.fullLabels)
+# exception = catcher.exception
+# # cls.assertTrue("Asked labels are not all available in the dataset" in exception)
+#
+#
+# class Test_getAllLabels(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.random_state = np.random.RandomState(42)
+# cls.fullLabels = cls.random_state.randint(0, 4, 10)
+# cls.availableLabelsNames = ["test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"]
+#
+# def test_simple(cls):
+# newLabels, newLabelsNames, usedIndices = get_multiview_db.get_all_labels(
+# cls.fullLabels, cls.availableLabelsNames)
+# cls.assertEqual(cls.availableLabelsNames, newLabelsNames)
+# np.testing.assert_array_equal(usedIndices, np.arange(10))
+# np.testing.assert_array_equal(newLabels, cls.fullLabels)
+#
+#
+# class Test_fillLabelNames(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.NB_CLASS = 2
+# cls.askedLabelsNames = ["test_label_1", "test_label_3"]
+# cls.random_state = np.random.RandomState(42)
+# cls.availableLabelsNames = ["test_label_" + str(_) for _ in range(40)]
+#
+# def test_simple(cls):
+# askedLabelsNames, askedLabelsNamesSet = get_multiview_db.fill_label_names(
+# cls.NB_CLASS,
+# cls.askedLabelsNames,
+# cls.random_state,
+# cls.availableLabelsNames)
+# cls.assertEqual(askedLabelsNames, cls.askedLabelsNames)
+# cls.assertEqual(askedLabelsNamesSet, set(cls.askedLabelsNames))
+#
+# def test_missing_labels_names(cls):
+# cls.NB_CLASS = 39
+# askedLabelsNames, askedLabelsNamesSet = get_multiview_db.fill_label_names(
+# cls.NB_CLASS,
+# cls.askedLabelsNames,
+# cls.random_state,
+# cls.availableLabelsNames)
+#
+# cls.assertEqual(askedLabelsNames,
+# ['test_label_1', 'test_label_3', 'test_label_35',
+# 'test_label_38', 'test_label_6', 'test_label_15',
+# 'test_label_32', 'test_label_28', 'test_label_8',
+# 'test_label_29', 'test_label_26', 'test_label_17',
+# 'test_label_19', 'test_label_10', 'test_label_18',
+# 'test_label_14', 'test_label_21', 'test_label_11',
+# 'test_label_34', 'test_label_0', 'test_label_27',
+# 'test_label_7', 'test_label_13', 'test_label_2',
+# 'test_label_39', 'test_label_23', 'test_label_4',
+# 'test_label_31', 'test_label_37', 'test_label_5',
+# 'test_label_36', 'test_label_25', 'test_label_33',
+# 'test_label_12', 'test_label_24', 'test_label_20',
+# 'test_label_22', 'test_label_9', 'test_label_16'])
+# cls.assertEqual(askedLabelsNamesSet, set(
+# ["test_label_" + str(_) for _ in range(30)] + [
+# "test_label_" + str(31 + _) for _ in range(9)]))
+#
+# def test_too_many_label_names(cls):
+# cls.NB_CLASS = 2
+# cls.askedLabelsNames = ["test_label_1", "test_label_3", "test_label_4",
+# "test_label_6"]
+# askedLabelsNames, askedLabelsNamesSet = get_multiview_db.fill_label_names(
+# cls.NB_CLASS,
+# cls.askedLabelsNames,
+# cls.random_state,
+# cls.availableLabelsNames)
+# cls.assertEqual(askedLabelsNames, ["test_label_3", "test_label_6"])
+# cls.assertEqual(askedLabelsNamesSet, {"test_label_3", "test_label_6"})
+#
+#
+# class Test_allAskedLabelsAreAvailable(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.askedLabelsNamesSet = {"test_label_1", "test_label_3"}
+# cls.availableLabelsNames = ["test_label_0", "test_label_1",
+# "test_label_2", "test_label_3"]
+#
+# def test_asked_available_labels(cls):
+# cls.assertTrue(
+# get_multiview_db.all_asked_labels_are_available(cls.askedLabelsNamesSet,
+# cls.availableLabelsNames))
+#
+# def test_asked_unavailable_label(cls):
+# cls.askedLabelsNamesSet = {"test_label_1", "test_label_3",
+# "chicken_is_heaven"}
+# cls.assertFalse(
+# get_multiview_db.all_asked_labels_are_available(cls.askedLabelsNamesSet,
+# cls.availableLabelsNames))
+#
+#
+# class Test_getClasses(unittest.TestCase):
+#
+# @classmethod
+# def setUpClass(cls):
+# cls.random_state = np.random.RandomState(42)
+#
+# def test_multiclass(cls):
+# labelsSet = get_multiview_db.get_classes(
+# cls.random_state.randint(0, 5, 30))
+# cls.assertEqual(labelsSet, {0, 1, 2, 3, 4})
+#
+# def test_biclass(cls):
+# labelsSet = get_multiview_db.get_classes(
+# cls.random_state.randint(0, 2, 30))
+# cls.assertEqual(labelsSet, {0, 1})
+#
+# def test_one_class(cls):
+# with cls.assertRaises(get_multiview_db.DatasetError) as catcher:
+# get_multiview_db.get_classes(np.zeros(30, dtype=int))
+# exception = catcher.exception
+# # cls.assertTrue("Dataset must have at least two different labels" in exception)
+#
+<<<<<<< HEAD
class Test_filterLabels(unittest.TestCase):
@classmethod
@@ -334,12 +664,14 @@ class Test_getClasses(unittest.TestCase):
exception = catcher.exception
# self.assertTrue("Dataset must have at least two different labels" in exception)
+=======
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
-class Test_getClassicDBhdf5(unittest.TestCase):
+class Test_get_classic_db_hdf5(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
+ def setUp(self):
rm_tmp()
+<<<<<<< HEAD
if not os.path.exists("multiview_platform/tests/tmp_tests"):
os.mkdir("multiview_platform/tests/tmp_tests")
cls.dataset_file = h5py.File(
@@ -446,45 +778,97 @@ class Test_getClassicDBhdf5(unittest.TestCase):
self.assertEqual(dataset_file.get("Metadata").attrs["datasetLength"], 10)
self.assertEqual(dataset_file.get("Metadata").attrs["nbView"], 4)
self.assertEqual(dataset_file.get("Metadata").attrs["nbClass"], 4)
+=======
+ os.mkdir(tmp_path)
+ self.rs = np.random.RandomState(42)
+ self.nb_view = 3
+ self.file_name = "test.hdf5"
+ self.nb_examples = 5
+ self.nb_class = 3
+ self.views = [self.rs.randint(0, 10, size=(self.nb_examples, 7))
+ for _ in range(self.nb_view)]
+ self.labels = self.rs.randint(0, self.nb_class, self.nb_examples)
+ self.dataset_file = h5py.File(os.path.join(tmp_path, self.file_name))
+ self.view_names = ["ViewN" + str(index) for index in
+ range(len(self.views))]
+ self.are_sparse = [False for _ in self.views]
+ for view_index, (view_name, view, is_sparse) in enumerate(
+ zip(self.view_names, self.views, self.are_sparse)):
+ view_dataset = self.dataset_file.create_dataset(
+ "View" + str(view_index),
+ view.shape,
+ data=view)
+ view_dataset.attrs["name"] = view_name
+ view_dataset.attrs["sparse"] = is_sparse
+ labels_dataset = self.dataset_file.create_dataset("Labels",
+ shape=self.labels.shape,
+ data=self.labels)
+ self.labels_names = [str(index) for index in np.unique(self.labels)]
+ labels_dataset.attrs["names"] = [label_name.encode()
+ for label_name in self.labels_names]
+ meta_data_grp = self.dataset_file.create_group("Metadata")
+ meta_data_grp.attrs["nbView"] = len(self.views)
+ meta_data_grp.attrs["nbClass"] = len(np.unique(self.labels))
+ meta_data_grp.attrs["datasetLength"] = len(self.labels)
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
- @classmethod
- def tearDownClass(cls):
- os.remove(
- tmp_path+"test_dataset_temp_view_label_select.hdf5")
- os.remove(tmp_path+"test_dataset.hdf5")
- dirs = os.listdir("multiview_platform/tests/tmp_tests")
- for dir in dirs:
- print(dir)
- os.rmdir("multiview_platform/tests/tmp_tests")
+ def test_simple(self):
+ dataset , labels_dictionary, dataset_name = get_multiview_db.get_classic_db_hdf5(
+ ["ViewN2"], tmp_path, self.file_name.split(".")[0],
+ self.nb_class, ["0", "2"],
+ self.rs, path_for_new=tmp_path)
+ self.assertEqual(dataset.nb_view, 1)
+ self.assertEqual(labels_dictionary,
+ {0: "0", 1: "2", 2:"1"})
+ self.assertEqual(dataset.get_nb_examples(), 5)
+ self.assertEqual(len(np.unique(dataset.get_labels())), 3)
-class Test_getClassicDBcsv(unittest.TestCase):
+ def test_all_views_asked(self):
+ dataset, labels_dictionary, dataset_name = get_multiview_db.get_classic_db_hdf5(
+ None, tmp_path, self.file_name.split(".")[0],
+ self.nb_class, ["0", "2"],
+ self.rs, path_for_new=tmp_path)
+ self.assertEqual(dataset.nb_view, 3)
+ self.assertEqual(dataset.get_view_dict(), {'ViewN0': 0, 'ViewN1': 1, 'ViewN2': 2})
- @classmethod
- def setUpClass(cls):
+ def test_asked_the_whole_dataset(self):
+ dataset, labels_dictionary, dataset_name = get_multiview_db.get_classic_db_hdf5(
+ ["ViewN2"], tmp_path, self.file_name.split(".")[0],
+ self.nb_class, ["0", "2"],
+ self.rs, path_for_new=tmp_path, full=True)
+ self.assertEqual(dataset.dataset, self.dataset_file)
+
+ def tearDown(self):
rm_tmp()
- if not os.path.exists("multiview_platform/tests/tmp_tests"):
- os.mkdir("multiview_platform/tests/tmp_tests")
- cls.pathF = tmp_path
- cls.NB_CLASS = 2
- cls.nameDB = "test_dataset"
- cls.askedLabelsNames = ["test_label_1", "test_label_3"]
- cls.random_state = np.random.RandomState(42)
- cls.views = ["test_view_1", "test_view_3"]
- np.savetxt(cls.pathF + cls.nameDB + "-labels-names.csv",
+
+
+class Test_get_classic_db_csv(unittest.TestCase):
+
+ def setUp(self):
+ rm_tmp()
+ os.mkdir(tmp_path)
+ self.pathF = tmp_path
+ self.NB_CLASS = 2
+ self.nameDB = "test_dataset"
+ self.askedLabelsNames = ["test_label_1", "test_label_3"]
+ self.random_state = np.random.RandomState(42)
+ self.views = ["test_view_1", "test_view_3"]
+ np.savetxt(self.pathF + self.nameDB + "-labels-names.csv",
np.array(["test_label_0", "test_label_1",
"test_label_2", "test_label_3"]), fmt="%s",
delimiter=",")
- np.savetxt(cls.pathF + cls.nameDB + "-labels.csv",
- cls.random_state.randint(0, 4, 10), delimiter=",")
- os.mkdir(cls.pathF + "Views")
- cls.datas = []
+ np.savetxt(self.pathF + self.nameDB + "-labels.csv",
+ self.random_state.randint(0, 4, 10), delimiter=",")
+ os.mkdir(self.pathF + "Views")
+ self.datas = []
for i in range(4):
- data = cls.random_state.randint(0, 100, (10, 20))
- np.savetxt(cls.pathF + "Views/test_view_" + str(i) + ".csv",
+ data = self.random_state.randint(0, 100, (10, 20))
+ np.savetxt(self.pathF + "Views/test_view_" + str(i) + ".csv",
data, delimiter=",")
- cls.datas.append(data)
+ self.datas.append(data)
+<<<<<<< HEAD
def test_simple(self):
dataset_file, labels_dictionary, dataset_name = get_multiview_db.get_classic_db_csv(
self.views, self.pathF, self.nameDB,
@@ -566,9 +950,23 @@ class Test_getClassicDBcsv(unittest.TestCase):
self.assertEqual(dataset_file.get("Metadata").attrs["datasetLength"], 10)
self.assertEqual(dataset_file.get("Metadata").attrs["nbView"], 4)
self.assertEqual(dataset_file.get("Metadata").attrs["nbClass"], 4)
+=======
+ def test_simple(cls):
+ dataset, labels_dictionary, dataset_name = get_multiview_db.get_classic_db_csv(
+ cls.views, cls.pathF, cls.nameDB,
+ cls.NB_CLASS, cls.askedLabelsNames,
+ cls.random_state, delimiter=",", path_for_new=tmp_path)
+ cls.assertEqual(dataset.nb_view, 2)
+ cls.assertEqual(dataset.get_view_dict(), {'test_view_1': 0, 'test_view_3': 1})
+ cls.assertEqual(labels_dictionary,
+ {0: "test_label_1", 1: "test_label_3"})
+ cls.assertEqual(dataset.get_nb_examples(), 3)
+ cls.assertEqual(dataset.get_nb_class(), 2)
+
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
@classmethod
- def tearDownClass(cls):
+ def tearDown(self):
for i in range(4):
os.remove(
tmp_path+"Views/test_view_" + str(
@@ -579,10 +977,8 @@ class Test_getClassicDBcsv(unittest.TestCase):
os.remove(tmp_path+"test_dataset-labels.csv")
os.remove(tmp_path+"test_dataset.hdf5")
os.remove(
- tmp_path+"test_dataset_temp_view_label_select.hdf5")
- for file in os.listdir("multiview_platform/tests/tmp_tests"): print(
- file)
- os.rmdir("multiview_platform/tests/tmp_tests")
+ tmp_path+"test_dataset_temp_filter.hdf5")
+ os.rmdir(tmp_path)
class Test_get_plausible_db_hdf5(unittest.TestCase):
diff --git a/multiview_platform/tests/test_utils/test_dataset.py b/multiview_platform/tests/test_utils/test_dataset.py
index 23ced56abe8a8102a0d6f4545b7536f1a49a8e6e..3c5c1c22645a6038c9bcef3c568b22d06d38177f 100644
--- a/multiview_platform/tests/test_utils/test_dataset.py
+++ b/multiview_platform/tests/test_utils/test_dataset.py
@@ -18,23 +18,24 @@ class Test_Dataset(unittest.TestCase):
cls.nb_view = 3
cls.file_name = "test.hdf5"
cls.nb_examples = 5
+ cls.nb_attr = 7
cls.nb_class = 3
- cls.views = [cls.rs.randint(0,10,size=(cls.nb_examples,7))
- for _ in range(cls.nb_view)]
- cls.labels = cls.rs.randint(0,cls.nb_class,cls.nb_examples)
+ cls.views = [cls.rs.randint(0, 10, size=(cls.nb_examples, cls.nb_attr))
+ for _ in range(cls.nb_view)]
+ cls.labels = cls.rs.randint(0, cls.nb_class, cls.nb_examples)
cls.dataset_file = h5py.File(os.path.join(tmp_path, cls.file_name))
cls.view_names = ["ViewN" + str(index) for index in range(len(cls.views))]
cls.are_sparse = [False for _ in cls.views]
for view_index, (view_name, view, is_sparse) in enumerate(
zip(cls.view_names, cls.views, cls.are_sparse)):
view_dataset = cls.dataset_file.create_dataset("View" + str(view_index),
- view.shape,
- data=view)
+ view.shape,
+ data=view)
view_dataset.attrs["name"] = view_name
view_dataset.attrs["sparse"] = is_sparse
labels_dataset = cls.dataset_file.create_dataset("Labels",
- shape=cls.labels.shape,
- data=cls.labels)
+ shape=cls.labels.shape,
+ data=cls.labels)
cls.labels_names = [str(index) for index in np.unique(cls.labels)]
labels_dataset.attrs["names"] = [label_name.encode()
for label_name in cls.labels_names]
@@ -46,11 +47,55 @@ class Test_Dataset(unittest.TestCase):
@classmethod
def tearDownClass(cls):
cls.dataset_file.close()
- rm_tmp()
- def test_simple(self):
+ def test_filter(self):
+ """Had to create a new dataset to aviod playing with the class one"""
+ file_name = "test_filter.hdf5"
+ dataset_file_filter = h5py.File(os.path.join(tmp_path, file_name))
+ for view_index, (view_name, view, is_sparse) in enumerate(
+ zip(self.view_names, self.views, self.are_sparse)):
+ view_dataset = dataset_file_filter.create_dataset(
+ "View" + str(view_index),
+ view.shape,
+ data=view)
+ view_dataset.attrs["name"] = view_name
+ view_dataset.attrs["sparse"] = is_sparse
+ labels_dataset = dataset_file_filter.create_dataset("Labels",
+ shape=self.labels.shape,
+ data=self.labels)
+ labels_dataset.attrs["names"] = [label_name.encode()
+ for label_name in self.labels_names]
+ meta_data_grp = dataset_file_filter.create_group("Metadata")
+ meta_data_grp.attrs["nbView"] = len(self.views)
+ meta_data_grp.attrs["nbClass"] = len(np.unique(self.labels))
+ meta_data_grp.attrs["datasetLength"] = len(self.labels)
+ dataset_object = dataset.Dataset(hdf5_file=dataset_file_filter)
+ dataset_object.filter(np.array([0, 1, 0]), ["0", "1"], [1, 2, 3],
+ ["ViewN0"], tmp_path)
+ self.assertEqual(dataset_object.nb_view, 1)
+ np.testing.assert_array_equal(dataset_object.get_labels(), [0, 1, 0])
+ dataset_object.dataset.close()
+ os.remove(os.path.join(tmp_path, "test_filter_temp_filter.hdf5"))
+ os.remove(os.path.join(tmp_path, "test_filter.hdf5"))
+
+ def test_for_hdf5_file(self):
dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ def test_from_scratch(self):
+ dataset_object = dataset.Dataset(views=self.views,
+ labels=self.labels,
+ are_sparse=self.are_sparse,
+ file_name="from_scratch"+self.file_name,
+ view_names=self.view_names,
+ path=tmp_path,
+ labels_names=self.labels_names)
+ nb_class = dataset_object.get_nb_class()
+ self.assertEqual(nb_class, self.nb_class)
+ example_indices = dataset_object.init_example_indces()
+ self.assertEqual(example_indices, range(self.nb_examples))
+ view = dataset_object.get_v(0)
+ np.testing.assert_array_equal(view, self.views[0])
+
def test_init_example_indices(self):
example_indices = dataset.Dataset(hdf5_file=self.dataset_file).init_example_indces()
self.assertEqual(example_indices, range(self.nb_examples))
@@ -69,20 +114,7 @@ class Test_Dataset(unittest.TestCase):
nb_class = dataset.Dataset(hdf5_file=self.dataset_file).get_nb_class([0])
self.assertEqual(nb_class, 1)
- def test_from_scratch(self):
- dataset_object = dataset.Dataset(views=self.views,
- labels=self.labels,
- are_sparse=self.are_sparse,
- file_name="from_scratch"+self.file_name,
- view_names=self.view_names,
- path=tmp_path,
- labels_names=self.labels_names)
- nb_class = dataset_object.get_nb_class()
- self.assertEqual(nb_class, self.nb_class)
- example_indices = dataset_object.init_example_indces()
- self.assertEqual(example_indices, range(self.nb_examples))
- view = dataset_object.get_v(0)
- np.testing.assert_array_equal(view, self.views[0])
+
def test_get_view_dict(self):
dataset_object = dataset.Dataset(views=self.views,
@@ -96,5 +128,117 @@ class Test_Dataset(unittest.TestCase):
"ViewN1": 1,
"ViewN2": 2,})
+<<<<<<< HEAD
if __name__ == '__main__':
- unittest.main()
\ No newline at end of file
+ unittest.main()
+=======
+ def test_get_label_names(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ raw_label_names = dataset_object.get_label_names(decode=False)
+ decoded_label_names = dataset_object.get_label_names()
+ restricted_label_names = dataset_object.get_label_names(example_indices=[3,4])
+ self.assertEqual(raw_label_names, [b'0', b'1', b'2'])
+ self.assertEqual(decoded_label_names, ['0', '1', '2'])
+ self.assertEqual(restricted_label_names, ['2'])
+
+ def test_get_nb_exmaples(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ nb_examples = dataset_object.get_nb_examples()
+ self.assertEqual(nb_examples, self.nb_examples)
+
+ def test_get_labels(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ labels = dataset_object.get_labels()
+ np.testing.assert_array_equal(labels, self.labels)
+ labels = dataset_object.get_labels([1,2,0])
+ np.testing.assert_array_equal(labels, self.labels[[1,2,0]])
+
+ def test_copy_view(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ new_dataset = h5py.File(os.path.join(tmp_path, "test_copy.hdf5"), "w")
+ dataset_object.copy_view(target_dataset=new_dataset,
+ source_view_name="ViewN0",
+ target_view_index=1)
+ self.assertIn("View1", list(new_dataset.keys()))
+ np.testing.assert_array_equal(dataset_object.get_v(0), new_dataset["View1"].value)
+ self.assertEqual(new_dataset["View1"].attrs["name"], "ViewN0")
+ new_dataset.close()
+ os.remove(os.path.join(tmp_path, "test_copy.hdf5"))
+
+ def test_get_name(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ self.assertEqual("test", dataset_object.get_name())
+
+ def test_select_labels(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ labels, label_names, indices = dataset_object.select_labels(["0", "2"])
+ np.testing.assert_array_equal(np.unique(labels), np.array([0,1]))
+ self.assertEqual(label_names, ["0","2"])
+
+ def test_check_selected_label_names(self):
+ dataset_object = dataset.Dataset(hdf5_file=self.dataset_file)
+ names = dataset_object.check_selected_label_names(nb_labels=2, random_state=self.rs)
+ self.assertEqual(names, ["1", "0"])
+ names = dataset_object.check_selected_label_names(selected_label_names=['0', '2'],
+ random_state=self.rs)
+ self.assertEqual(names, ["0", "2"])
+
+ def test_select_views_and_labels(self):
+ file_name = "test_filter.hdf5"
+ dataset_file_select = h5py.File(os.path.join(tmp_path, file_name))
+ for view_index, (view_name, view, is_sparse) in enumerate(
+ zip(self.view_names, self.views, self.are_sparse)):
+ view_dataset = dataset_file_select.create_dataset(
+ "View" + str(view_index),
+ view.shape,
+ data=view)
+ view_dataset.attrs["name"] = view_name
+ view_dataset.attrs["sparse"] = is_sparse
+ labels_dataset = dataset_file_select.create_dataset("Labels",
+ shape=self.labels.shape,
+ data=self.labels)
+ labels_dataset.attrs["names"] = [label_name.encode()
+ for label_name in self.labels_names]
+ meta_data_grp = dataset_file_select.create_group("Metadata")
+ meta_data_grp.attrs["nbView"] = len(self.views)
+ meta_data_grp.attrs["nbClass"] = len(np.unique(self.labels))
+ meta_data_grp.attrs["datasetLength"] = len(self.labels)
+ dataset_object = dataset.Dataset(hdf5_file=dataset_file_select)
+ names = dataset_object.select_views_and_labels(nb_labels=2, view_names=["ViewN0"], random_state=self.rs, path_for_new=tmp_path)
+ self.assertEqual(names, {0: '2', 1: '1'})
+ self.assertEqual(dataset_object.nb_view, 1)
+ dataset_object.dataset.close()
+ os.remove(os.path.join(tmp_path, "test_filter_temp_filter.hdf5"))
+ os.remove(os.path.join(tmp_path, "test_filter.hdf5"))
+
+ def test_add_gaussian_noise(self):
+ file_name = "test_noise.hdf5"
+ dataset_file_select = h5py.File(os.path.join(tmp_path, file_name))
+ limits = np.zeros((self.nb_attr, 2))
+ limits[:, 1] += 100
+ meta_data_grp = dataset_file_select.create_group("Metadata")
+ for view_index, (view_name, view, is_sparse) in enumerate(
+ zip(self.view_names, self.views, self.are_sparse)):
+ view_dataset = dataset_file_select.create_dataset(
+ "View" + str(view_index),
+ view.shape,
+ data=view)
+ view_dataset.attrs["name"] = view_name
+ view_dataset.attrs["sparse"] = is_sparse
+ meta_data_grp.create_dataset("View"+str(view_index)+"_limits", data= limits)
+ labels_dataset = dataset_file_select.create_dataset("Labels",
+ shape=self.labels.shape,
+ data=self.labels)
+ labels_dataset.attrs["names"] = [label_name.encode()
+ for label_name in self.labels_names]
+ meta_data_grp.attrs["nbView"] = len(self.views)
+ meta_data_grp.attrs["nbClass"] = len(np.unique(self.labels))
+ meta_data_grp.attrs["datasetLength"] = len(self.labels)
+ dataset_object = dataset.Dataset(hdf5_file=dataset_file_select)
+ dataset_object.add_gaussian_noise(self.rs, tmp_path)
+ dataset_object.dataset.close()
+ os.remove(os.path.join(tmp_path, "test_noise_noised.hdf5"))
+ os.remove(os.path.join(tmp_path, "test_noise.hdf5"))
+
+
+>>>>>>> 66129965ae7c38b4bdb4ae657369ab24357070cd
diff --git a/multiview_platform/tests/test_utils/test_hyper_parameter_search.py b/multiview_platform/tests/test_utils/test_hyper_parameter_search.py
index 2fafecce33551ba555bfe01f97d11988f2198e43..edc164fa75046d197e0d9b597ec35c4b200b609d 100644
--- a/multiview_platform/tests/test_utils/test_hyper_parameter_search.py
+++ b/multiview_platform/tests/test_utils/test_hyper_parameter_search.py
@@ -7,6 +7,7 @@ from sklearn.model_selection import StratifiedKFold
from ..utils import rm_tmp, tmp_path
+from ...mono_multi_view_classifiers.utils.dataset import Dataset
from ...mono_multi_view_classifiers.utils import hyper_parameter_search
from ...mono_multi_view_classifiers.multiview_classifiers import weighted_linear_early_fusion
@@ -27,10 +28,12 @@ class Test_randomized_search(unittest.TestCase):
view0 = cls.dataset_file.create_dataset("View0",
data=cls.view0_data)
view0.attrs["sparse"] = False
+ view0.attrs["name"] = "ViewN0"
cls.view1_data = cls.random_state.randint(1, 10, size=(10, 4))
view1 = cls.dataset_file.create_dataset("View1",
data=cls.view1_data)
view1.attrs["sparse"] = False
+ view1.attrs["name"] = "ViewN1"
metaDataGrp = cls.dataset_file.create_group("Metadata")
metaDataGrp.attrs["nbView"] = 2
metaDataGrp.attrs["nbClass"] = 2
@@ -41,6 +44,7 @@ class Test_randomized_search(unittest.TestCase):
"splitter": "best"}
cls.k_folds = StratifiedKFold(n_splits=3, random_state=cls.random_state)
cls.learning_indices = np.array([1,2,3,4, 5,6,7,8,9])
+ cls.dataset = Dataset(hdf5_file=cls.dataset_file)
@classmethod
def tearDownClass(cls):
@@ -53,7 +57,7 @@ class Test_randomized_search(unittest.TestCase):
def test_simple(self):
best_params, test_folds_preds = hyper_parameter_search.randomized_search(
- self.dataset_file, self.labels.value, "multiview", self.random_state, tmp_path,
+ self.dataset, self.labels.value, "multiview", self.random_state, tmp_path,
weighted_linear_early_fusion, "WeightedLinearEarlyFusion", self.k_folds,
1, ["accuracy_score", None], 2, {}, learning_indices=self.learning_indices)
diff --git a/multiview_platform/tests/utils.py b/multiview_platform/tests/utils.py
index cc77a9be8b4b960ceb86506aca2ab2c1c4e2f76e..5a373caeb54edfdcb4212e4d3b4da5bf2953a8f7 100644
--- a/multiview_platform/tests/utils.py
+++ b/multiview_platform/tests/utils.py
@@ -1,7 +1,12 @@
import os
+import numpy as np
+import h5py
+
+from ..mono_multi_view_classifiers.utils.dataset import Dataset
tmp_path = "multiview_platform/tests/tmp_tests/"
+test_dataset = Dataset(hdf5_file=h5py.File("multiview_platform/tests/test_database.hdf5", "r"))
def rm_tmp():
try:
@@ -10,3 +15,35 @@ def rm_tmp():
os.rmdir(tmp_path)
except:
pass
+
+
+def gen_test_dataset(random_state=np.random.RandomState(42)):
+ dataset_file = h5py.File("test_database.hdf5", "w")
+ view_names = ["ViewN0", "ViewN1", "ViewN2"]
+ views = [random_state.randint(0,100,(5,6))
+ for _ in range(len(view_names))]
+ labels = random_state.randint(0,2, 5)
+ label_names = ["yes", "no"]
+ for view_index, (view_name, view) in enumerate(
+ zip(view_names, views)):
+ view_dataset = dataset_file.create_dataset("View" + str(view_index),
+ view.shape,
+ data=view)
+ view_dataset.attrs["name"] = view_name
+ view_dataset.attrs["sparse"] = False
+ labels_dataset = dataset_file.create_dataset("Labels",
+ shape=labels.shape,
+ data=labels)
+ labels_dataset.attrs["names"] = [label_name.encode()
+ if not isinstance(label_name, bytes)
+ else label_name
+ for label_name in label_names]
+ meta_data_grp = dataset_file.create_group("Metadata")
+ meta_data_grp.attrs["nbView"] = len(views)
+ meta_data_grp.attrs["nbClass"] = len(np.unique(labels))
+ meta_data_grp.attrs["datasetLength"] = len(labels)
+ dataset_file.close()
+
+
+if __name__ == "__main__":
+ gen_test_dataset()