diff --git a/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQ.py b/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQ.py
index 7edf2bf666916978bd778e94b17826cd9b99bbe6..73133a3e093bf17d4ced59ed01c60efba9e76156 100644
--- a/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQ.py
+++ b/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQ.py
@@ -54,7 +54,7 @@ class MinCqLearner(BaseEstimator, ClassifierMixin):
Kernel coefficient for 'rbf' and 'poly'.
If gamma is 0.0 then 1/n_features will be used instead.
"""
- def __init__(self, mu, voters_type, n_stumps_per_attribute=10, kernel='rbf', degree=3, gamma=0.0):
+ def __init__(self, mu, voters_type, n_stumps_per_attribute=10, kernel='rbf', degree=3, gamma=0.0, self_complemented=True):
assert mu > 0 and mu <= 1, "MinCqLearner: mu parameter must be in (0, 1]"
self.mu = mu
self.voters_type = voters_type
@@ -63,6 +63,7 @@ class MinCqLearner(BaseEstimator, ClassifierMixin):
self.degree = degree
self.gamma = gamma
self.log = False
+ self.self_complemented = self_complemented
self.majority_vote = None
self.qp = None
@@ -114,7 +115,7 @@ class MinCqLearner(BaseEstimator, ClassifierMixin):
elif self.kernel == 'rbf':
voters_generator = KernelVotersGenerator(rbf_kernel, gamma=gamma)
- voters = voters_generator.generate(X, y_reworked)
+ voters = voters_generator.generate(X, y_reworked, self_complemented=self.self_complemented)
if self.log:
logging.info("MinCq training started...")
@@ -524,13 +525,14 @@ class KernelVotersGenerator(VotersGenerator):
class MinCQ(MinCqLearner, BaseMonoviewClassifier):
- def __init__(self, random_state=None, mu=0.01, epsilon=1e-06, **kwargs):
+ def __init__(self, random_state=None, mu=0.01, self_complemented=True , **kwargs):
super(MinCQ, self).__init__(mu=mu,
voters_type='stumps',
- n_stumps_per_attribute =10
+ n_stumps_per_attribute =10,
+ self_complemented=self_complemented
)
self.param_names = ["mu"]
- self.distribs = [CustomUniform(loc=0.5, state=1.0, multiplier="e-"),
+ self.distribs = [CustomUniform(loc=0.5, state=2.0, multiplier="e-"),
]
self.classed_params = []
self.weird_strings = {}
diff --git a/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQGraalpy.py b/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQGraalpy.py
new file mode 100644
index 0000000000000000000000000000000000000000..b638c6fbd47584dc1894c85fe84db83b77b7c9c8
--- /dev/null
+++ b/multiview_platform/MonoMultiViewClassifiers/MonoviewClassifiers/MinCQGraalpy.py
@@ -0,0 +1,393 @@
+# -*- coding: utf-8 -*-
+"""MinCq algorithm.
+
+Related papers:
+[1] From PAC-Bayes Bounds to Quadratic Programs for Majority Votes (Laviolette et al., 2011)
+[2] Risk Bounds for the Majority Vote: From a PAC-Bayesian Analysis to a Learning Algorithm (Germain et al., 2015)
+
+"""
+from __future__ import print_function, division, absolute_import
+import logging
+from operator import xor
+
+import numpy as np
+from scipy.linalg import sqrtm
+from scipy.spatial.distance import pdist, squareform
+from sklearn.metrics.pairwise import rbf_kernel
+from sklearn.utils.validation import check_X_y
+from sklearn.ensemble import VotingClassifier
+from sklearn.manifold import SpectralEmbedding
+from sklearn.utils.graph import graph_laplacian
+from sklearn.preprocessing import LabelEncoder
+
+
+from ..Monoview.Additions.BoostUtils import ConvexProgram, StumpsClassifiersGenerator
+from ..Monoview.MonoviewUtils import BaseMonoviewClassifier, CustomUniform
+from ..Metrics import zero_one_loss
+
+# logger = logging.getLogger('MinCq')
+
+class MinCqClassifier(VotingClassifier):
+ """
+ Base MinCq algorithm learner. See [1, 2].
+ This version is an attempt of creating a more general version of MinCq, that handles multiclass classfication.
+ For binary classification, use RegularizedMinCqClassifer.
+
+ Parameters
+ ----------
+ mu : float
+ The fixed value of the first moment of the margin.
+
+ """
+ def __init__(self, estimators_generator=None, estimators=None, mu=0.001, omega=0.5, use_binary=False, zeta=0, gamma=1, n_neighbors=5):
+ if estimators is None:
+ estimators = []
+
+ super().__init__(estimators=estimators, voting='soft')
+ self.estimators_generator = estimators_generator
+ self.mu = mu
+ self.omega = omega
+ self.use_binary = use_binary
+ self.zeta = zeta
+ self.gamma = gamma
+ self.n_neighbors = n_neighbors
+
+ def fit(self, X, y):
+ """Fit the estimators and learn the weights.
+
+ Parameters
+ ----------
+ X : array-like, shape = [n_samples, n_features]
+ Training vectors, where n_samples is the number of samples and
+ n_features is the number of features.
+ y : array-like, shape = [n_samples]
+ Target values. If y is a masked-array (numpy.ma), the masked values are unlabeled examples.
+
+ Returns
+ -------
+ self : object
+
+ """
+ # Validations
+ assert 0 < self.mu <= 1, "MinCqClassifier: mu parameter must be in (0, 1]"
+ assert xor(bool(self.estimators_generator), bool(self.estimators)), "MinCqClassifier: exactly one of estimator_generator or estimators must be used."
+ X, y = check_X_y(X, y)
+
+ # Fit the estimators using VotingClassifier's fit method. This will also fit a LabelEncoder that can be
+ # used to "normalize" labels (0, 1, 2, ...). In the case of binary classification, the two classes will be 0 and 1.
+ # First, ensure that the weights are reset to None (as cloning a VotingClassifier keeps the weights)
+ self.weights = None
+ # TODO: Ensure estimators can deal with masked arrays
+
+ # If we use an estimator generator, use the data-dependant estimator generator to generate them, and fit again.
+ if self.estimators:
+ super().fit(X, y)
+
+ else:
+ self.le_ = LabelEncoder()
+ self.le_.fit(y)
+
+ if isinstance(y, np.ma.MaskedArray):
+ transformed_y = np.ma.MaskedArray(self.le_.transform(y), y.mask)
+ else:
+ transformed_y = self.le_.transform(y)
+
+ self.estimators_generator.fit(X, transformed_y)
+ self.estimators = [('ds{}'.format(i), estimator) for i, estimator in enumerate(self.estimators_generator.estimators_)]
+ super().fit(X, y)
+
+ # We clean the estimators attribute (as we do not want it to be cloned later)
+ # self.estimators_ = []
+
+ # logger.info("Training started...")
+ # logger.info("Training dataset shape: {}".format(str(np.shape(X))))
+ # logger.info("Number of voters: {}".format(len(self.estimators_)))
+
+ # Preparation and resolution of the quadratic program
+ # logger.info("Preparing and solving QP...")
+ self.weights = self._solve(X, y)
+
+ return self
+
+ # def evaluate_metrics(self, X, y, metrics_list=None, functions_list=None):
+ # if metrics_list is None:
+ # metrics_list = [zero_one_loss]
+ #
+ # if functions_list is None:
+ # functions_list = []
+ # else:
+ # raise NotImplementedError
+ #
+ # # Predict, evaluate metrics.
+ # predictions = self.predict(X)
+ # metrics_results = {metric.__name__: metric(y, predictions) for metric in metrics_list}
+ #
+ # metrics_dataframe = ResultsDataFrame([metrics_results])
+ # return metrics_dataframe
+
+ def _binary_classification_matrix(self, X):
+ probas = self.transform(X)
+ predicted_labels = np.argmax(probas, axis=2)
+ predicted_labels[predicted_labels == 0] = -1
+ values = np.max(probas, axis=2)
+ return (predicted_labels * values).T
+
+ def _multiclass_classification_matrix(self, X, y):
+ probas = self.transform(X).swapaxes(0, 1)
+ matrix = probas[np.arange(probas.shape[0]), :, y]
+
+ return (matrix - self.omega)
+
+ def _solve(self, X, y):
+ y = self.le_.transform(y)
+
+ if self.use_binary:
+ assert len(self.le_.classes_) == 2
+
+ # TODO: Review the number of labeled examples when adding back the support for transductive learning.
+ classification_matrix = self._binary_classification_matrix(X)
+
+ # We use {-1, 1} labels.
+ binary_labels = np.copy(y)
+ binary_labels[y == 0] = -1
+
+ multi_matrix = binary_labels.reshape((len(binary_labels), 1)) * classification_matrix
+
+ else:
+ multi_matrix = self._multiclass_classification_matrix(X, y)
+
+ n_examples, n_voters = np.shape(multi_matrix)
+ ftf = 1.0 / n_examples * multi_matrix.T.dot(multi_matrix)
+ yf = np.mean(multi_matrix, axis=0)
+
+ # Objective function.
+ objective_matrix = 2 * ftf
+ objective_vector = None
+
+ # Equality constraints (first moment of the margin equal to mu, Q sums to one)
+ equality_matrix = np.vstack((yf.reshape((1, n_voters)), np.ones((1, n_voters))))
+ equality_vector = np.array([self.mu, 1.0])
+
+ # Lower and upper bounds, no quasi-uniformity.
+ lower_bound = 0.0
+ # TODO: In the case of binary classification, no upper bound will give
+ # bad results. Using 1/n works, as it brings back the l_infinity
+ # regularization normally given by the quasi-uniformity constraint.
+ # upper_bound = 2.0/n_voters
+ upper_bound = None
+
+ weights = self._solve_qp(objective_matrix, objective_vector, equality_matrix, equality_vector, lower_bound, upper_bound)
+
+ # Keep learning information for further use.
+ self.learner_info_ = {}
+
+ # We count the number of non-zero weights, including the implicit voters.
+ # TODO: Verify how we define non-zero weights here, could be if the weight is near 1/2n.
+ n_nonzero_weights = np.sum(np.asarray(weights) > 1e-12)
+ n_nonzero_weights += np.sum(np.asarray(weights) < 1.0 / len(self.estimators_) - 1e-12)
+ self.learner_info_.update(n_nonzero_weights=n_nonzero_weights)
+
+ return weights
+
+ def _solve_qp(self, objective_matrix, objective_vector, equality_matrix, equality_vector, lower_bound, upper_bound):
+ try:
+ qp = ConvexProgram()
+ qp.quadratic_func, qp.linear_func = objective_matrix, objective_vector
+ qp.add_equality_constraints(equality_matrix, equality_vector)
+ qp.add_lower_bound(lower_bound)
+ qp.add_upper_bound(upper_bound)
+ return qp.solve()
+
+ except Exception:
+ # logger.warning("Error while solving the quadratic program.")
+ raise
+
+
+class RegularizedBinaryMinCqClassifier(MinCqClassifier):
+ """MinCq, version published in [1] and [2], where the regularization comes from the enforced quasi-uniformity
+ of the posterior distributino on the symmetric hypothesis space. This version only works with {-1, 1} labels.
+
+ [1] From PAC-Bayes Bounds to Quadratic Programs for Majority Votes (Laviolette et al., 2011)
+ [2] Risk Bounds for the Majority Vote: From a PAC-Bayesian Analysis to a Learning Algorithm (Germain et al., 2015)
+
+ """
+ def fit(self, X, y):
+ # We first fit and learn the weights.
+ super().fit(X, y)
+
+ # Validations
+ if isinstance(y, np.ma.MaskedArray):
+ assert len(self.classes_[np.where(np.logical_not(self.classes_.mask))]) == 2, "RegularizedBinaryMinCqClassifier: only supports binary classification."
+ else:
+ assert len(self.classes_), "RegularizedBinaryMinCqClassifier: only supports binary classification."
+
+ # Then we "reverse" the negative weights and their associated voter's output.
+ for i, weight in enumerate(self.weights):
+ if weight < 0:
+ # logger.debug("Reversing decision of a binary voter")
+ self.weights[i] *= -1
+ self.estimators_[i].reverse_decision()
+
+ return self
+
+ def _solve(self, X, y):
+ if isinstance(y, np.ma.MaskedArray):
+ y = np.ma.MaskedArray(self.le_.transform(y), y.mask)
+ else:
+ y = self.le_.transform(y)
+
+ classification_matrix = self._binary_classification_matrix(X)
+ n_examples, n_voters = np.shape(classification_matrix)
+
+ if self.zeta == 0:
+ ftf = classification_matrix.T.dot(classification_matrix)
+ else:
+ I = np.eye(n_examples)
+ L = build_laplacian(X, n_neighbors=self.n_neighbors)
+ ftf = classification_matrix.T.dot(I + (self.zeta / n_examples) * L).dot(classification_matrix)
+
+ # We use {-1, 1} labels.
+ binary_labels = np.ma.copy(y)
+ binary_labels[np.ma.where(y == 0)] = -1
+
+ # Objective function.
+ ftf_mean = np.mean(ftf, axis=1)
+ objective_matrix = 2.0 / n_examples * ftf
+ objective_vector = -1.0 / n_examples * ftf_mean.T
+
+ # Equality constraint: first moment of the margin fixed to mu, only using labeled examples.
+ if isinstance(y, np.ma.MaskedArray):
+ labeled = np.where(np.logical_not(y.mask))[0]
+ binary_labels = binary_labels[labeled]
+ else:
+ labeled = range(len(y))
+
+ yf = binary_labels.T.dot(classification_matrix[labeled])
+ yf_mean = np.mean(yf)
+ equality_matrix = 2.0 / len(labeled) * yf
+ equality_vector = self.mu + 1.0 / len(labeled) * yf_mean
+
+ # Lower and upper bounds (quasi-uniformity constraints)
+ lower_bound = 0.0
+ upper_bound = 1.0 / n_voters
+
+ weights = self._solve_qp(objective_matrix, objective_vector, equality_matrix, equality_vector, lower_bound, upper_bound)
+
+ # Keep learning information for further use.
+ self.learner_info_ = {}
+
+ # We count the number of non-zero weights, including the implicit voters.
+ # TODO: Verify how we define non-zero weights here, could be if the weight is near 1/2n.
+ n_nonzero_weights = np.sum(np.asarray(weights) > 1e-12)
+ n_nonzero_weights += np.sum(np.asarray(weights) < 1.0 / len(self.estimators_) - 1e-12)
+ self.learner_info_.update(n_nonzero_weights=n_nonzero_weights)
+
+ # Conversion of the weights of the n first voters to weights on the implicit 2n voters.
+ # See Section 7.1 of [2] for an explanation.
+ return np.array([2 * q - 1.0 / len(self.estimators_) for q in weights])
+
+ # def evaluate_metrics(self, X, y, metrics_list=None, functions_list=None):
+ # if metrics_list is None:
+ # metrics_list = [zero_one_loss]
+ #
+ # if functions_list is None:
+ # functions_list = []
+ #
+ # # Transductive setting: we only predict the X for labeled y
+ # if isinstance(y, np.ma.MaskedArray):
+ # labeled = np.where(np.logical_not(y.mask))[0]
+ # X = np.array(X[labeled])
+ # y = np.array(y[labeled])
+ #
+ # # Predict, evaluate metrics.
+ # predictions = self.predict(X)
+ # metrics_results = {metric.__name__: metric(y, predictions) for metric in metrics_list}
+ #
+ # # TODO: Repair in the case of non-{-1, 1} labels.
+ # assert set(y) == {-1, 1}
+ # classification_matrix = self._binary_classification_matrix(X)
+ #
+ # for function in functions_list:
+ # metrics_results[function.__name__] = function(classification_matrix, y, self.weights)
+ #
+ # metrics_dataframe = ResultsDataFrame([metrics_results])
+ # return metrics_dataframe
+
+
+def build_laplacian(X, n_neighbors=None):
+ clf = SpectralEmbedding(n_neighbors=n_neighbors)
+ clf.fit(X)
+ w = clf.affinity_matrix_
+ laplacian = graph_laplacian(w, normed=True)
+ return laplacian
+
+
+class MinCQGraalpy(RegularizedBinaryMinCqClassifier, BaseMonoviewClassifier):
+
+ def __init__(self, random_state=None, mu=0.01, self_complemented=True, n_stumps_per_attribute=10 , **kwargs):
+ super(MinCQGraalpy, self).__init__(mu=mu,
+ estimators_generator=StumpsClassifiersGenerator(n_stumps_per_attribute=n_stumps_per_attribute, self_complemented=self_complemented),
+ )
+ self.param_names = ["mu"]
+ self.distribs = [CustomUniform(loc=0.5, state=2.0, multiplier="e-"),
+ ]
+ self.classed_params = []
+ self.weird_strings = {}
+ if "nbCores" not in kwargs:
+ self.nbCores = 1
+ else:
+ self.nbCores = kwargs["nbCores"]
+
+ def canProbas(self):
+ """Used to know if the classifier can return label probabilities"""
+ return True
+
+ def getInterpret(self, directory, y_test):
+ interpret_string = ""
+ # interpret_string += "Train C_bound value : "+str(self.cbound_train)
+ # y_rework = np.copy(y_test)
+ # y_rework[np.where(y_rework==0)] = -1
+ # interpret_string += "\n Test c_bound value : "+str(self.majority_vote.cbound_value(self.x_test, y_rework))
+ return interpret_string
+
+ def get_name_for_fusion(self):
+ return "MCG"
+
+
+def formatCmdArgs(args):
+ """Used to format kwargs for the parsed args"""
+ kwargsDict = {"mu":args.MCG_mu,
+ "n_stumps_per_attribute":args.MCG_stumps}
+ return kwargsDict
+
+
+def paramsToSet(nIter, randomState):
+ """Used for weighted linear early fusion to generate random search sets"""
+ paramsSet = []
+ for _ in range(nIter):
+ paramsSet.append({})
+ return paramsSet
+
+
+# if __name__ == '__main__':
+# # Example usage.
+# from sklearn.datasets import load_iris
+# from sklearn.cross_validation import train_test_split
+# from graalpy.utils.majority_vote import StumpsClassifiersGenerator
+#
+# # Load data, change {0, 1, 2} labels to {-1, 1}
+# iris = load_iris()
+# iris.target[np.where(iris.target == 0)] = -1
+# iris.target[np.where(iris.target == 2)] = 1
+# x_train, x_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state=42)
+#
+# # Fit MinCq
+# clf = RegularizedBinaryMinCqClassifier(estimators_generator=StumpsClassifiersGenerator())
+# clf.fit(x_train, y_train)
+#
+# # Compare the best score of individual classifiers versus the score of the learned majority vote.
+# print("Best training risk of individual voters: {:.4f}".format(1 - max([e.score(x_train, y_train) for e in clf.estimators_])))
+# print("Training risk of the majority vote outputted by MinCq: {:.4f}".format(1 - clf.score(x_train, y_train)))
+# print()
+# print("Best testing risk of individual voters: {:.4f}".format(1 - max([e.score(x_test, y_test) for e in clf.estimators_])))
+# print("Testing risk of the majority vote outputted by MinCq: {:.4f}".format(1 - clf.score(x_test, y_test)))
diff --git a/multiview_platform/MonoMultiViewClassifiers/utils/execution.py b/multiview_platform/MonoMultiViewClassifiers/utils/execution.py
index 43b2e1d8f4dbfe69187658913d810b2b2bd2ad19..9a3a91d49af6345c2c0254d404cee54918a6f8b0 100644
--- a/multiview_platform/MonoMultiViewClassifiers/utils/execution.py
+++ b/multiview_platform/MonoMultiViewClassifiers/utils/execution.py
@@ -138,7 +138,7 @@ def parseTheArgs(arguments):
groupAdaboostGraalpy.add_argument('--AdG_n_iter', metavar='INT', type=int,
action='store',
help='Number of estimators',
- default=2)
+ default=100)
groupAdaboostGraalpy.add_argument('--AdG_stumps', metavar='INT', type=int,
action='store',
help='Number of stumps inthe pregenerated dataset',
@@ -182,7 +182,7 @@ def parseTheArgs(arguments):
groupCQBoost.add_argument('--CQB_mu', metavar='FLOAT', type=float, action='store',
help='Set the mu parameter for CQBoost', default=0.001)
groupCQBoost.add_argument('--CQB_epsilon', metavar='FLOAT', type=float, action='store',
- help='Set the epsilon parameter for CQBoost', default=1e-08)
+ help='Set the epsilon parameter for CQBoost', default=1e-06)
groupCQBoost.add_argument('--CQB_stumps', metavar='INT', type=int,
action='store',
help='Set the number of stumps for CQBoost',
@@ -190,7 +190,7 @@ def parseTheArgs(arguments):
groupCQBoostv2 = parser.add_argument_group('CQBoostv2 arguments')
groupCQBoostv2.add_argument('--CQB2_mu', metavar='FLOAT', type=float, action='store',
- help='Set the mu parameter for CQBoostv2', default=0.001)
+ help='Set the mu parameter for CQBoostv2', default=0.002)
groupCQBoostv2.add_argument('--CQB2_epsilon', metavar='FLOAT', type=float, action='store',
help='Set the epsilon parameter for CQBoostv2', default=1e-08)
@@ -238,17 +238,26 @@ def parseTheArgs(arguments):
help='Set the n_estimators_parameter for Gradient Boosting',
default=100)
- groupMinCQ = parser.add_argument_group(
- 'MinCQ arguments')
- groupMinCQ.add_argument('--MCQ_mu', metavar='float', type=int,
- action='store',
- help='Set the mu_parameter for MinCQ',
- default=1e-3)
+ groupMinCQ = parser.add_argument_group('MinCQ arguments')
+ groupMinCQ.add_argument('--MCQ_mu', metavar='FLOAT', type=float,
+ action='store',
+ help='Set the mu_parameter for MinCQ',
+ default=0.05)
groupMinCQ.add_argument('--MCQ_stumps', metavar='INT', type=int,
action='store',
help='Set the n_stumps_per_attribute parameter for MinCQ',
default=1)
+ groupMinCQGraalpy = parser.add_argument_group('MinCQGraalpy arguments')
+ groupMinCQGraalpy.add_argument('--MCG_mu', metavar='FLOAT', type=float,
+ action='store',
+ help='Set the mu_parameter for MinCQGraalpy',
+ default=0.05)
+ groupMinCQGraalpy.add_argument('--MCG_stumps', metavar='INT', type=int,
+ action='store',
+ help='Set the n_stumps_per_attribute parameter for MinCQGraalpy',
+ default=1)
+
groupQarBoostv3 = parser.add_argument_group('QarBoostv3 arguments')
groupQarBoostv3.add_argument('--QarB3_mu', metavar='FLOAT', type=float, action='store',
help='Set the mu parameter for QarBoostv3', default=0.001)