Skip to content
Snippets Groups Projects

Resolve "Experiment pipeline"

Merged Charly Lamothe requested to merge 12-experiment-pipeline into master
4 files
+ 46
37
Compare changes
  • Side-by-side
  • Inline
Files
4
from bolsonaro.data.dataset import Dataset
from bolsonaro.data.dataset_parameters import DatasetParameters
from bolsonaro.data.task import Task
from bolsonaro.utils import change_binary_func_load
@@ -9,10 +10,22 @@ from sklearn.datasets import fetch_olivetti_faces, fetch_20newsgroups, \
fetch_covtype, fetch_rcv1, fetch_kddcup99, fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import random
class DatasetLoader(object):
DEFAULT_DATASET_NAME = 'boston'
DEFAULT_NORMALIZE_D = False
DEFAULT_DATASET_NORMALIZER = 'standard'
DEFAULT_FOREST_SIZE = 100
DEFAULT_EXTRACTED_FOREST_SIZE = 10
DEFAULT_DEV_SIZE = 0.2
DEFAULT_TEST_SIZE = 0.2
DEFAULT_RANDOM_SEED_NUMBER = 1
DEFAULT_SUBSETS_USED = 'train,dev'
DEFAULT_NORMALIZE_WEIGHTS = False
@staticmethod
def load(dataset_parameters):
name = dataset_parameters.name
@@ -67,7 +80,7 @@ class DatasetLoader(object):
raise ValueError("Unsupported dataset '{}'".format(name))
if X is None:
X, y = dataset_loading_func()
X, y = dataset_loading_func(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=dataset_parameters.test_size,
@@ -93,3 +106,20 @@ class DatasetLoader(object):
return Dataset(task, X_train,
X_dev, X_test, y_train, y_dev, y_test)
@staticmethod
def load_default(dataset_name, seed):
begin_random_seed_range = 1
end_random_seed_range = 2000
seed = seed if seed else random.randint(begin_random_seed_range, end_random_seed_range)
dataset_parameters = DatasetParameters(
name=dataset_name,
test_size=DatasetLoader.DEFAULT_TEST_SIZE,
dev_size=DatasetLoader.DEFAULT_DEV_SIZE,
random_state=seed,
dataset_normalizer=DatasetLoader.DEFAULT_DATASET_NORMALIZER
)
return DatasetLoader.load(dataset_parameters)
Loading