Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
B
bolsonaro
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Luc Giffon
bolsonaro
Merge requests
!15
Resolve "Adding new datasets"
Code
Review changes
Check out branch
Download
Patches
Plain diff
Merged
Resolve "Adding new datasets"
17-adding-new-datasets
into
master
Overview
0
Commits
39
Pipelines
0
Changes
2
Merged
Leo Bouscarrat
requested to merge
17-adding-new-datasets
into
master
5 years ago
Overview
0
Commits
39
Pipelines
0
Changes
2
Expand
Closes
#17 (closed)
Edited
5 years ago
by
Charly Lamothe
0
0
Merge request reports
Viewing commit
b5bbdccd
Prev
Next
Show latest version
2 files
+
2
−
5
Inline
Compare changes
Side-by-side
Inline
Show whitespace changes
Show one file at a time
Files
2
Search (e.g. *.vue) (Ctrl+P)
b5bbdccd
Removing inexisting parameter to fetch_openml
· b5bbdccd
Léo Bouscarrat
authored
5 years ago
code/bolsonaro/data/dataset_loader.py
+
25
−
4
Options
from
bolsonaro.data.dataset
import
Dataset
from
bolsonaro.data.dataset_parameters
import
DatasetParameters
from
bolsonaro.data.task
import
Task
from
bolsonaro.utils
import
change_binary_func_load
from
bolsonaro.utils
import
change_binary_func_load
,
change_binary_func_openml
from
sklearn.datasets
import
load_boston
,
load_iris
,
load_diabetes
,
\
load_digits
,
load_linnerud
,
load_wine
,
load_breast_cancer
from
sklearn.datasets
import
fetch_olivetti_faces
,
fetch_20newsgroups
,
\
fetch_20newsgroups_vectorized
,
fetch_lfw_people
,
fetch_lfw_pairs
,
\
fetch_covtype
,
fetch_rcv1
,
fetch_kddcup99
,
fetch_california_housing
fetch_covtype
,
fetch_rcv1
,
fetch_kddcup99
,
fetch_california_housing
,
\
fetch_openml
from
sklearn.model_selection
import
train_test_split
from
sklearn
import
preprocessing
import
random
@@ -30,13 +31,15 @@ class DatasetLoader(object):
dataset_names
=
[
'
boston
'
,
'
iris
'
,
'
diabetes
'
,
'
digits
'
,
'
linnerud
'
,
'
wine
'
,
'
breast_cancer
'
,
'
olivetti_faces
'
,
'
20newsgroups_vectorized
'
,
'
lfw_people
'
,
'
lfw_pairs
'
,
'
covtype
'
,
'
rcv1
'
,
'
california_housing
'
,
'
diamonds
'
]
'
lfw_pairs
'
,
'
covtype
'
,
'
rcv1
'
,
'
california_housing
'
,
'
diamonds
'
,
'
steel-plates
'
,
'
kr-vs-kp
'
,
'
kin8nm
'
,
'
spambase
'
,
'
musk
'
,
'
gamma
'
]
dataset_seed_numbers
=
{
'
boston
'
:
15
,
'
iris
'
:
15
,
'
diabetes
'
:
15
,
'
digits
'
:
5
,
'
linnerud
'
:
15
,
'
wine
'
:
15
,
'
breast_cancer
'
:
15
,
'
olivetti_faces
'
:
15
,
'
20newsgroups_vectorized
'
:
3
,
'
lfw_people
'
:
3
,
'
lfw_pairs
'
:
3
,
'
covtype
'
:
3
,
'
rcv1
'
:
3
,
'
california_housing
'
:
3
,
'
diamonds
'
:
15
}
'
diamonds
'
:
15
,
'
steel-plates
'
:
15
,
'
kr-vs-kp
'
:
15
,
'
kin8nm
'
:
15
,
'
spambase
'
:
15
,
'
musk
'
:
15
,
'
gamma
'
:
15
}
@staticmethod
def
load
(
dataset_parameters
):
@@ -103,6 +106,24 @@ class DatasetLoader(object):
df
[
'
clarity
'
]
=
label_clarity
.
fit_transform
(
df
[
'
clarity
'
])
X
,
y
=
df
.
drop
([
'
price
'
],
axis
=
1
),
df
[
'
price
'
]
task
=
Task
.
REGRESSION
elif
name
==
'
steel-plates
'
:
dataset_loading_func
=
change_binary_func_openml
(
'
steel-plates-fault
'
)
task
=
Task
.
BINARYCLASSIFICATION
elif
name
==
'
kr-vs-kp
'
:
dataset_loading_func
=
change_binary_func_openml
(
'
kr-vs-kp
'
)
task
=
Task
.
BINARYCLASSIFICATION
elif
name
==
'
kin8nm
'
:
X
,
y
=
fetch_openml
(
'
kin8nm
'
,
return_X_y
=
True
)
task
=
Task
.
REGRESSION
elif
name
==
'
spambase
'
:
dataset_loading_func
=
change_binary_func_openml
(
'
spambase
'
)
task
=
Task
.
BINARYCLASSIFICATION
elif
name
==
'
musk
'
:
dataset_loading_func
=
change_binary_func_openml
(
'
musk
'
)
task
=
Task
.
BINARYCLASSIFICATION
elif
name
==
'
gamma
'
:
dataset_loading_func
=
change_binary_func_openml
(
'
MagicTelescope
'
)
task
=
Task
.
BINARYCLASSIFICATION
else
:
raise
ValueError
(
"
Unsupported dataset
'
{}
'"
.
format
(
name
))
Loading