Resolve "Experiment pipeline"

Closes #12 (closed)

code/bolsonaro/trainer.py

 from bolsonaro.models.model_raw_results import ModelRawResults
+from bolsonaro.models.omp_forest_regressor import OmpForestRegressor
+from bolsonaro.models.omp_forest_classifier import OmpForestBinaryClassifier, OmpForestMulticlassClassifier
+from bolsonaro.models.similarity_forest_regressor import SimilarityForestRegressor
 from bolsonaro.error_handling.logger_factory import LoggerFactory
+from bolsonaro.data.task import Task
 from . import LOG_PATH
 
+from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
+from sklearn.metrics import mean_squared_error, accuracy_score
 import time
 import datetime
 import numpy as np
@@ -12,16 +18,41 @@ class Trainer(object):
     Class capable of fitting any model object to some prepared data then evaluate and save results through the `train` method.
     """
 
-    def __init__(self, dataset):
+    def __init__(self, dataset, regression_score_metric=mean_squared_error, classification_score_metric=accuracy_score,
+        base_regression_score_metric=mean_squared_error, base_classification_score_metric=accuracy_score):
         """
 
         :param dataset: Object with X_train, y_train, X_dev, y_dev, X_test and Y_test attributes
         """
         self._dataset = dataset
         self._logger = LoggerFactory.create(LOG_PATH, __name__)
+        self._regression_score_metric = regression_score_metric
+        self._classification_score_metric = classification_score_metric
+        self._base_regression_score_metric = base_regression_score_metric
+        self._base_classification_score_metric = base_classification_score_metric
+        self._score_metric_name = regression_score_metric.__name__ if dataset.task == Task.REGRESSION \
+            else classification_score_metric.__name__
+        self._base_score_metric_name = base_regression_score_metric.__name__ if dataset.task == Task.REGRESSION \
+            else base_classification_score_metric.__name__
 
-    def init(self, model):
-        if model.models_parameters.subsets_used == 'train,dev':
+    @property
+    def score_metric_name(self):
+        return self._score_metric_name
+
+    @property
+    def base_score_metric_name(self):
+        return self._base_score_metric_name
+
+    def init(self, model, subsets_used='train,dev'):
+        if type(model) in [RandomForestRegressor, RandomForestClassifier]:
+            if subsets_used == 'train,dev':
+                self._X_forest = self._dataset.X_train
+                self._y_forest = self._dataset.y_train
+            else:
+                self._X_forest = np.concatenate([self._dataset.X_train, self._dataset.X_dev])
+                self._y_forest = np.concatenate([self._dataset.y_train, self._dataset.y_dev])    
+            self._logger.debug('Fitting the forest on train subset')
+        elif model.models_parameters.subsets_used == 'train,dev':
             self._X_forest = self._dataset.X_train
             self._y_forest = self._dataset.y_train
             self._X_omp = self._dataset.X_dev
@@ -43,43 +74,77 @@ class Trainer(object):
 
     def train(self, model):
         """
-        :param model: Object with
+        :param model: An instance of either RandomForestRegressor, RandomForestClassifier, OmpForestRegressor,
+            OmpForestBinaryClassifier, OmpForestMulticlassClassifier.
         :return:
         """
 
         self._logger.debug('Training model using train set...')
         self._begin_time = time.time()
+        if type(model) in [RandomForestRegressor, RandomForestClassifier]:
+            model.fit(
+                X=self._X_forest,
+                y=self._y_forest
+            )
+        else:
             model.fit(
-            X_forest=self._X_forest,
-            y_forest=self._y_forest,
-            X_omp=self._X_omp,
-            y_omp=self._y_omp
+                self._X_forest,
+                self._y_forest,
+                self._X_omp,
+                self._y_omp
             )
         self._end_time = time.time()
 
+    def __score_func(self, model, X, y_true):
+        if type(model) in [OmpForestRegressor, RandomForestRegressor, SimilarityForestRegressor]:
+            y_pred = model.predict(X)
+            result = self._regression_score_metric(y_true, y_pred)
+        elif type(model) in [OmpForestBinaryClassifier, OmpForestMulticlassClassifier, RandomForestClassifier]:
+            y_pred = model.predict(X)
+            if type(model) is OmpForestBinaryClassifier:
+                y_pred = y_pred.round()
+            result = self._classification_score_metric(y_true, y_pred)
+        return result
+
+    def __score_func_base(self, model, X, y_true):
+        if type(model) == OmpForestRegressor:
+            y_pred = model.predict_base_estimator(X)
+            result = self._base_regression_score_metric(y_true, y_pred)
+        elif type(model) in [OmpForestBinaryClassifier, OmpForestMulticlassClassifier]:
+            y_pred = model.predict_base_estimator(X)
+            result = self._base_classification_score_metric(y_true, y_pred)
+        elif type(model) == RandomForestClassifier:
+            y_pred = model.predict(X)
+            result = self._base_classification_score_metric(y_true, y_pred)
+        elif type(model) in [RandomForestRegressor, SimilarityForestRegressor]:
+            y_pred = model.predict(X)
+            result = self._base_regression_score_metric(y_true, y_pred)
+        return result
+
     def compute_results(self, model, models_dir):
         """
         :param model: Object with
         :param models_dir: Where the results will be saved
         """
         results = ModelRawResults(
-            model_object=model,
+            model_object='',
             training_time=self._end_time - self._begin_time,
             datetime=datetime.datetime.now(),
-            train_score=model.score(self._dataset.X_train, self._dataset.y_train),
-            dev_score=model.score(self._dataset.X_dev, self._dataset.y_dev),
-            test_score=model.score(self._dataset.X_test, self._dataset.y_test),
-            score_metric=model.DEFAULT_SCORE_METRIC, # TODO: resolve the used metric in a proper way
-            train_score_regressor=model.score_base_estimator(self._dataset.X_train, self._dataset.y_train),
-            dev_score_regressor=model.score_base_estimator(self._dataset.X_dev, self._dataset.y_dev),
-            test_score_regressor=model.score_base_estimator(self._dataset.X_test, self._dataset.y_test)
+            train_score=self.__score_func(model, self._dataset.X_train, self._dataset.y_train),
+            dev_score=self.__score_func(model, self._dataset.X_dev, self._dataset.y_dev),
+            test_score=self.__score_func(model, self._dataset.X_test, self._dataset.y_test),
+            train_score_base=self.__score_func_base(model, self._dataset.X_train, self._dataset.y_train),
+            dev_score_base=self.__score_func_base(model, self._dataset.X_dev, self._dataset.y_dev),
+            test_score_base=self.__score_func_base(model, self._dataset.X_test, self._dataset.y_test),
+            score_metric=self._score_metric_name,
+            base_score_metric=self._base_score_metric_name
         )
         results.save(models_dir)
-        self._logger.info("Base performance on test: {}".format(results.test_score_regressor))
+        self._logger.info("Base performance on test: {}".format(results.test_score_base))
         self._logger.info("Performance on test: {}".format(results.test_score))
 
-        self._logger.info("Base performance on train: {}".format(results.train_score_regressor))
+        self._logger.info("Base performance on train: {}".format(results.train_score_base))
         self._logger.info("Performance on train: {}".format(results.train_score))
 
-        self._logger.info("Base performance on dev: {}".format(results.dev_score_regressor))
+        self._logger.info("Base performance on dev: {}".format(results.dev_score_base))
         self._logger.info("Performance on dev: {}".format(results.dev_score))