From 9a9a3bff3b76cfc2855fc6b2b689b9b99d64c745 Mon Sep 17 00:00:00 2001
From: Charly LAMOTHE <lamothe.c@intlocal.univ-amu.fr>
Date: Sat, 9 Nov 2019 18:15:15 +0100
Subject: [PATCH] Introduce the notion of "stage" in the experiments (see TODO
comment in compute_results.py file. Add example of experiment configuration
file tree in experiments/boston/stage3 for stage 3 (unoptimized parameters
due to missing stage 1 and 2 results).
---
code/compute_results.py | 15 ++++++++-------
.../boston_train+dev,train+dev.json | 18 ++++++++++++++++++
...boston_train+dev,train+dev_normalize-D.json | 18 ++++++++++++++++++
...+dev_normalize-D_weights-normalization.json | 18 ++++++++++++++++++
...in+dev,train+dev_weights-normalization.json | 18 ++++++++++++++++++
.../stage3/train,dev/boston_train,dev.json | 18 ++++++++++++++++++
.../boston_train,dev_normalize-D.json | 18 ++++++++++++++++++
...,dev_normalize-D_weights-normalization.json | 18 ++++++++++++++++++
...boston_train,dev_weights-normalization.json | 18 ++++++++++++++++++
.../boston_train,train+dev.json | 18 ++++++++++++++++++
.../boston_train,train+dev_normalize-D.json | 18 ++++++++++++++++++
...+dev_normalize-D_weights-normalization.json | 18 ++++++++++++++++++
..._train,train+dev_weights-normalization.json | 18 ++++++++++++++++++
13 files changed, 224 insertions(+), 7 deletions(-)
create mode 100644 experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json
create mode 100644 experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json
create mode 100644 experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json
create mode 100644 experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json
create mode 100644 experiments/boston/stage3/train,dev/boston_train,dev.json
create mode 100644 experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json
create mode 100644 experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json
create mode 100644 experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json
create mode 100644 experiments/boston/stage3/train,train+dev/boston_train,train+dev.json
create mode 100644 experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json
create mode 100644 experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json
create mode 100644 experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json
diff --git a/code/compute_results.py b/code/compute_results.py
index dad9992..0f26eb1 100644
--- a/code/compute_results.py
+++ b/code/compute_results.py
@@ -119,13 +119,14 @@ if __name__ == "__main__":
"""
TODO:
For each dataset:
- 0) A figure for the selection of the best base forest model hyperparameters (best vs default/random hyperparams)
- 1) A figure for the selection of the best dataset normalization method
- 2) A figure for the selection of the best combination of dataset: normalization vs D normalization vs weights normalization
- 3) A figure for the selection of the most relevant subsets combination: train,dev vs train+dev,train+dev vs train,train+dev
- 4) A figure to finally compare the perf of our approach using the previous selected parameters vs the baseline vs other papers
-
- 2)
+ Stage 1) A figure for the selection of the best base forest model hyperparameters (best vs default/random hyperparams)
+ Stage 2) A figure for the selection of the best dataset normalization method
+ Stage 3) A figure for the selection of the best combination of dataset: normalization vs D normalization vs weights normalization
+ Stage 4) A figure for the selection of the most relevant subsets combination: train,dev vs train+dev,train+dev vs train,train+dev
+ Stage 5) A figure for the selection of the best extracted forest size?
+ Stage 6) A figure to finally compare the perf of our approach using the previous selected parameters vs the baseline vs other papers
+
+ Stage 3)
In all axis:
- untrained forest
- trained base forest (straight line cause it doesn't depend on the number of extracted trees)
diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json
new file mode 100644
index 0000000..b6dd49c
--- /dev/null
+++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train+dev,train+dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json
new file mode 100644
index 0000000..8d50e19
--- /dev/null
+++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train+dev,train+dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json
new file mode 100644
index 0000000..2e7b19e
--- /dev/null
+++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train+dev,train+dev",
+ "normalize_weights": true
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json
new file mode 100644
index 0000000..c0fa623
--- /dev/null
+++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train+dev,train+dev",
+ "normalize_weights": true
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,dev/boston_train,dev.json b/experiments/boston/stage3/train,dev/boston_train,dev.json
new file mode 100644
index 0000000..0ffac35
--- /dev/null
+++ b/experiments/boston/stage3/train,dev/boston_train,dev.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json
new file mode 100644
index 0000000..d7f1c2e
--- /dev/null
+++ b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json
new file mode 100644
index 0000000..824133a
--- /dev/null
+++ b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,dev",
+ "normalize_weights": true
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json b/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json
new file mode 100644
index 0000000..45e9173
--- /dev/null
+++ b/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,dev",
+ "normalize_weights": true
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json
new file mode 100644
index 0000000..4da1e6d
--- /dev/null
+++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,train+dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json
new file mode 100644
index 0000000..ccc9bef
--- /dev/null
+++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,train+dev",
+ "normalize_weights": false
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json
new file mode 100644
index 0000000..93c0082
--- /dev/null
+++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": true,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,train+dev",
+ "normalize_weights": true
+}
\ No newline at end of file
diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json
new file mode 100644
index 0000000..ed3bf08
--- /dev/null
+++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json
@@ -0,0 +1,18 @@
+{
+ "dataset_name": "boston",
+ "normalize_D": false,
+ "dataset_normalizer": "standard",
+ "forest_size": 100,
+ "extracted_forest_size": [
+ 10,
+ 20,
+ 30
+ ],
+ "models_dir": ".\\models",
+ "dev_size": 0.2,
+ "test_size": 0.2,
+ "random_seed_number": 3,
+ "seeds": null,
+ "subsets_used": "train,train+dev",
+ "normalize_weights": true
+}
\ No newline at end of file
--
GitLab