diff --git a/code/compute_results.py b/code/compute_results.py index dad99922deaa802bd2e794b96f4f3b0f42c97239..0f26eb101a1910577593223a166e19b495f73d85 100644 --- a/code/compute_results.py +++ b/code/compute_results.py @@ -119,13 +119,14 @@ if __name__ == "__main__": """ TODO: For each dataset: - 0) A figure for the selection of the best base forest model hyperparameters (best vs default/random hyperparams) - 1) A figure for the selection of the best dataset normalization method - 2) A figure for the selection of the best combination of dataset: normalization vs D normalization vs weights normalization - 3) A figure for the selection of the most relevant subsets combination: train,dev vs train+dev,train+dev vs train,train+dev - 4) A figure to finally compare the perf of our approach using the previous selected parameters vs the baseline vs other papers - - 2) + Stage 1) A figure for the selection of the best base forest model hyperparameters (best vs default/random hyperparams) + Stage 2) A figure for the selection of the best dataset normalization method + Stage 3) A figure for the selection of the best combination of dataset: normalization vs D normalization vs weights normalization + Stage 4) A figure for the selection of the most relevant subsets combination: train,dev vs train+dev,train+dev vs train,train+dev + Stage 5) A figure for the selection of the best extracted forest size? + Stage 6) A figure to finally compare the perf of our approach using the previous selected parameters vs the baseline vs other papers + + Stage 3) In all axis: - untrained forest - trained base forest (straight line cause it doesn't depend on the number of extracted trees) diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json new file mode 100644 index 0000000000000000000000000000000000000000..b6dd49c4a7f9ef9b8ae97c1ac578d35f0a47c171 --- /dev/null +++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train+dev,train+dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json new file mode 100644 index 0000000000000000000000000000000000000000..8d50e1964663c6f4cd88efc2e7c85e4e19b2ced3 --- /dev/null +++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train+dev,train+dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..2e7b19ec64d0d36048022df069377e3cb3b0d88e --- /dev/null +++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_normalize-D_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train+dev,train+dev", + "normalize_weights": true +} \ No newline at end of file diff --git a/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..c0fa623dadbb7a142c9f3916428e225dea94ddba --- /dev/null +++ b/experiments/boston/stage3/train+dev,train+dev/boston_train+dev,train+dev_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train+dev,train+dev", + "normalize_weights": true +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,dev/boston_train,dev.json b/experiments/boston/stage3/train,dev/boston_train,dev.json new file mode 100644 index 0000000000000000000000000000000000000000..0ffac35eb43a7568bb14a85010e538b094490b72 --- /dev/null +++ b/experiments/boston/stage3/train,dev/boston_train,dev.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json new file mode 100644 index 0000000000000000000000000000000000000000..d7f1c2e8427278615e76b7dc734c8936bef6fe57 --- /dev/null +++ b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..824133af36f3c226799c3d5d025f3cfab9fbd421 --- /dev/null +++ b/experiments/boston/stage3/train,dev/boston_train,dev_normalize-D_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,dev", + "normalize_weights": true +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json b/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..45e91739f838f6c1dbcc94e6dd5da136eca08f1d --- /dev/null +++ b/experiments/boston/stage3/train,dev/boston_train,dev_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,dev", + "normalize_weights": true +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json new file mode 100644 index 0000000000000000000000000000000000000000..4da1e6d4b9b10d620b23adee3a6b1719078da01d --- /dev/null +++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,train+dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc9befa778ccac3eb5d9efeebaa3fb8f1624c61 --- /dev/null +++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,train+dev", + "normalize_weights": false +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..93c0082c477841a765b3feb4bda6d4529ee14dcc --- /dev/null +++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_normalize-D_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,train+dev", + "normalize_weights": true +} \ No newline at end of file diff --git a/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..ed3bf0823c1d2c7b6da82f9554b492820ca9c638 --- /dev/null +++ b/experiments/boston/stage3/train,train+dev/boston_train,train+dev_weights-normalization.json @@ -0,0 +1,18 @@ +{ + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": 100, + "extracted_forest_size": [ + 10, + 20, + 30 + ], + "models_dir": ".\\models", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 3, + "seeds": null, + "subsets_used": "train,train+dev", + "normalize_weights": true +} \ No newline at end of file