Skip to content
Snippets Groups Projects

Resolve "Adding new datasets"

Merged Leo Bouscarrat requested to merge 17-adding-new-datasets into master
Compare and Show latest version
80 files
+ 1796
1213
Compare changes
  • Side-by-side
  • Inline
Files
80
from bolsonaro.data.dataset import Dataset
from bolsonaro.data.dataset_parameters import DatasetParameters
from bolsonaro.data.task import Task
from bolsonaro.utils import change_binary_func_load
from bolsonaro.utils import change_binary_func_load, change_binary_func_openml
from sklearn.datasets import load_boston, load_iris, load_diabetes, \
load_digits, load_linnerud, load_wine, load_breast_cancer
from sklearn.datasets import fetch_olivetti_faces, fetch_20newsgroups, \
fetch_20newsgroups_vectorized, fetch_lfw_people, fetch_lfw_pairs, \
fetch_covtype, fetch_rcv1, fetch_kddcup99, fetch_california_housing
fetch_covtype, fetch_rcv1, fetch_kddcup99, fetch_california_housing, \
fetch_openml
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
import random
@@ -30,13 +31,15 @@ class DatasetLoader(object):
dataset_names = ['boston', 'iris', 'diabetes', 'digits', 'linnerud', 'wine',
'breast_cancer', 'olivetti_faces', '20newsgroups_vectorized', 'lfw_people',
'lfw_pairs', 'covtype', 'rcv1', 'california_housing', 'diamonds']
'lfw_pairs', 'covtype', 'rcv1', 'california_housing', 'diamonds', 'steel-plates',
'kr-vs-kp', 'kin8nm', 'spambase', 'musk', 'gamma']
dataset_seed_numbers = {'boston':15, 'iris':15, 'diabetes':15, 'digits':5,
'linnerud':15, 'wine':15, 'breast_cancer':15, 'olivetti_faces':15,
'20newsgroups_vectorized':3, 'lfw_people':3,
'lfw_pairs':3, 'covtype':3, 'rcv1':3, 'california_housing':3,
'diamonds': 15}
'diamonds': 15, 'steel-plates': 15, 'kr-vs-kp': 15, 'kin8nm': 15,
'spambase': 15, 'musk': 15, 'gamma': 15}
@staticmethod
def load(dataset_parameters):
@@ -103,6 +106,24 @@ class DatasetLoader(object):
df['clarity'] = label_clarity.fit_transform(df['clarity'])
X, y = df.drop(['price'], axis=1), df['price']
task = Task.REGRESSION
elif name == 'steel-plates':
dataset_loading_func = change_binary_func_openml('steel-plates-fault')
task = Task.BINARYCLASSIFICATION
elif name == 'kr-vs-kp':
dataset_loading_func = change_binary_func_openml('kr-vs-kp')
task = Task.BINARYCLASSIFICATION
elif name == 'kin8nm':
X, y = fetch_openml('kin8nm', return_X_y=True)
task = Task.REGRESSION
elif name == 'spambase':
dataset_loading_func = change_binary_func_openml('spambase')
task = Task.BINARYCLASSIFICATION
elif name == 'musk':
dataset_loading_func = change_binary_func_openml('musk')
task = Task.BINARYCLASSIFICATION
elif name == 'gamma':
dataset_loading_func = change_binary_func_openml('MagicTelescope')
task = Task.BINARYCLASSIFICATION
else:
raise ValueError("Unsupported dataset '{}'".format(name))
Loading