diff --git a/code/bolsonaro/trainer.py b/code/bolsonaro/trainer.py index 218ba76ddbe7b9d8b68eb39b6e202c8d061107d0..09562958a0c1a58c8a82dde4cbbcd538f02c54d0 100644 --- a/code/bolsonaro/trainer.py +++ b/code/bolsonaro/trainer.py @@ -122,7 +122,7 @@ class Trainer(object): :param models_dir: Where the results will be saved """ results = ModelRawResults( - model_object=model, + model_object='', training_time=self._end_time - self._begin_time, datetime=datetime.datetime.now(), train_score=self.__score_func(model, self._dataset.X_train, self._dataset.y_train), diff --git a/code/compute_results.py b/code/compute_results.py index 42d90ca8285560ee4a8c765ffcb2aacaef718d0f..d0dadfcc44ec1a74f728a5d802dfb89b2e4e7b55 100644 --- a/code/compute_results.py +++ b/code/compute_results.py @@ -122,7 +122,7 @@ if __name__ == "__main__": DEFAULT_MODELS_DIR = os.environ["project_dir"] + os.sep + 'models' parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--stage', nargs='?', type=int, required=True, help='Specify the stage number among [1, 4].') + parser.add_argument('--stage', nargs='?', type=int, required=True, help='Specify the stage number among [1, 5].') parser.add_argument('--experiment_ids', nargs='+', type=int, required=True, help='Compute the results of the specified experiment id(s).' + \ 'stage=1: {{base_with_params}} {{random_with_params}} {{omp_with_params}} {{base_wo_params}} {{random_wo_params}} {{omp_wo_params}}' + \ 'stage=2: {{no_normalization}} {{normalize_D}} {{normalize_weights}} {{normalize_D_and_weights}}' + \ @@ -132,8 +132,8 @@ if __name__ == "__main__": parser.add_argument('--models_dir', nargs='?', type=str, default=DEFAULT_MODELS_DIR, help='The output directory of the trained models.') args = parser.parse_args() - if args.stage not in list(range(1, 5)): - raise ValueError('stage must be a supported stage id (i.e. [1, 4]).') + if args.stage not in list(range(1, 6)): + raise ValueError('stage must be a supported stage id (i.e. [1, 5]).') logger = LoggerFactory.create(LOG_PATH, os.path.basename(__file__)) @@ -327,6 +327,48 @@ if __name__ == "__main__": xlabel='Number of trees extracted', ylabel=experiments_score_metric, title='Loss values of {}\nusing different training subsets'.format(args.dataset_name))""" + elif args.stage == 4: + if len(args.experiment_ids) != 3: + raise ValueError('In the case of stage 4, the number of specified experiment ids must be 3.') + + # Retreive the extracted forest sizes number used in order to have a base forest axis as long as necessary + extracted_forest_sizes_number = retreive_extracted_forest_sizes_number(args.models_dir, args.experiment_ids[1]) + + # base_with_params + logger.info('Loading base_with_params experiment scores...') + base_with_params_train_scores, base_with_params_dev_scores, base_with_params_test_scores, \ + base_with_params_experiment_score_metric = \ + extract_scores_across_seeds_and_forest_size(args.models_dir, args.results_dir, args.experiment_ids[0], + extracted_forest_sizes_number) + # random_with_params + logger.info('Loading random_with_params experiment scores...') + random_with_params_train_scores, random_with_params_dev_scores, random_with_params_test_scores, \ + with_params_extracted_forest_sizes, random_with_params_experiment_score_metric = \ + extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, args.experiment_ids[1]) + # omp_with_params + logger.info('Loading omp_with_params experiment scores...') + omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores, _, \ + omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes( + args.models_dir, args.results_dir, args.experiment_ids[2]) + + # Sanity check on the metrics retreived + if not (base_with_params_experiment_score_metric == random_with_params_experiment_score_metric + == omp_with_params_experiment_score_metric): + raise ValueError('Score metrics of all experiments must be the same.') + experiments_score_metric = base_with_params_experiment_score_metric + + output_path = os.path.join(args.results_dir, args.dataset_name, 'stage4') + pathlib.Path(output_path).mkdir(parents=True, exist_ok=True) + + Plotter.plot_stage2_losses( + file_path=output_path + os.sep + 'losses.png', + all_experiment_scores=[base_with_params_test_scores, random_with_params_test_scores, + omp_with_params_test_scores], + all_labels=['base', 'random', 'omp'], + x_value=with_params_extracted_forest_sizes, + xlabel='Number of trees extracted', + ylabel=experiments_score_metric, + title='Loss values of {}\nusing best params of previous stages'.format(args.dataset_name)) else: raise ValueError('This stage number is not supported yet, but it will be!') diff --git a/code/train.py b/code/train.py index 10b083cbd4cd4b3a286ce9a667313c10c201d700..0931191ac4872a45ec615d65420886128540f1a1 100644 --- a/code/train.py +++ b/code/train.py @@ -122,6 +122,12 @@ python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --save_ python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --save_experiment_configuration 3 train-dev_train-dev_subset --extracted_forest_size_stop=0.05 --subsets_used train+dev,train+dev python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --save_experiment_configuration 3 train-train-dev_subset --extracted_forest_size_stop=0.05 --subsets_used train,train+dev python code/compute_results.py --stage 3 --experiment_ids 11 12 13 --dataset_name=california_housing + +Command lines example for stage 4: +python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --extraction_strategy=none --save_experiment_configuration 4 none_with_params --extracted_forest_size_stop=0.05 +python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --extraction_strategy=random --save_experiment_configuration 4 random_with_params --extracted_forest_size_stop=0.05 +python code/train.py --dataset_name=california_housing --seeds 1 2 3 4 5 --save_experiment_configuration 4 omp_with_params --extracted_forest_size_stop=0.05 --subsets_used train+dev,train+dev +python code/compute_results.py --stage 4 --experiment_ids 1 2 3 --dataset_name=california_housing """ if __name__ == "__main__": load_dotenv(find_dotenv('.env')) @@ -229,8 +235,8 @@ if __name__ == "__main__": if args.save_experiment_configuration: if len(args.save_experiment_configuration) != 2: raise ValueError('save_experiment_configuration must have two parameters.') - elif int(args.save_experiment_configuration[0]) not in list(range(1, 5)): - raise ValueError('save_experiment_configuration first parameter must be a supported stage id (i.e. [1, 4]).') + elif int(args.save_experiment_configuration[0]) not in list(range(1, 6)): + raise ValueError('save_experiment_configuration first parameter must be a supported stage id (i.e. [1, 5]).') output_experiment_stage_path = os.path.join(args.experiment_configuration_path, args.dataset_name, 'stage' + args.save_experiment_configuration[0]) pathlib.Path(output_experiment_stage_path).mkdir(parents=True, exist_ok=True) diff --git a/experiments/20newsgroups_vectorized/stage4/none_with_params.json b/experiments/20newsgroups_vectorized/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..c23c044f58f35717e44515d450309d5ff36bbb3d --- /dev/null +++ b/experiments/20newsgroups_vectorized/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "20newsgroups_vectorized", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/20newsgroups_vectorized/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 10, + 21, + 31, + 42, + 52, + 63, + 73, + 84, + 94, + 104, + 115, + 125, + 136, + 146, + 157, + 167, + 177, + 188, + 198, + 209, + 219, + 230, + 240, + 251, + 261, + 271, + 282, + 292, + 303, + 313 + ] +} \ No newline at end of file diff --git a/experiments/20newsgroups_vectorized/stage4/omp_with_params.json b/experiments/20newsgroups_vectorized/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..7ed6e55fce6443d6911d1e8c7d73402c8898bd4b --- /dev/null +++ b/experiments/20newsgroups_vectorized/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "20newsgroups_vectorized", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/20newsgroups_vectorized/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 10, + 21, + 31, + 42, + 52, + 63, + 73, + 84, + 94, + 104, + 115, + 125, + 136, + 146, + 157, + 167, + 177, + 188, + 198, + 209, + 219, + 230, + 240, + 251, + 261, + 271, + 282, + 292, + 303, + 313 + ] +} \ No newline at end of file diff --git a/experiments/20newsgroups_vectorized/stage4/random_with_params.json b/experiments/20newsgroups_vectorized/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..87f22cdaff1e325338721d3d198122471eb8378f --- /dev/null +++ b/experiments/20newsgroups_vectorized/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "20newsgroups_vectorized", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/20newsgroups_vectorized/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 10, + 21, + 31, + 42, + 52, + 63, + 73, + 84, + 94, + 104, + 115, + 125, + 136, + 146, + 157, + 167, + 177, + 188, + 198, + 209, + 219, + 230, + 240, + 251, + 261, + 271, + 282, + 292, + 303, + 313 + ] +} \ No newline at end of file diff --git a/experiments/boston/stage4/none_with_params.json b/experiments/boston/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..9c8dccaeb8890615712d02c9316e95322b9197a5 --- /dev/null +++ b/experiments/boston/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/boston/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/boston/stage4/omp_with_params.json b/experiments/boston/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..fdf5673f9443bff04a2642e12f03b375ed44649d --- /dev/null +++ b/experiments/boston/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/boston/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/boston/stage4/random_with_params.json b/experiments/boston/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..281036b9de515380e302a46a1fb82cfa224db43a --- /dev/null +++ b/experiments/boston/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "boston", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/boston/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/breast_cancer/stage4/none_with_params.json b/experiments/breast_cancer/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..fa938bcc737b2e133386a0cc5dc22d13783cebf2 --- /dev/null +++ b/experiments/breast_cancer/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "breast_cancer", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/breast_cancer/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 1, + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 30, + 31, + 32, + 34, + 35, + 36, + 37, + 39 + ] +} \ No newline at end of file diff --git a/experiments/breast_cancer/stage4/omp_with_params.json b/experiments/breast_cancer/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..16e4f61bf1a9e90156fd68aca7016b0919a80dac --- /dev/null +++ b/experiments/breast_cancer/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "breast_cancer", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/breast_cancer/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 1, + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 30, + 31, + 32, + 34, + 35, + 36, + 37, + 39 + ] +} \ No newline at end of file diff --git a/experiments/breast_cancer/stage4/random_with_params.json b/experiments/breast_cancer/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..9990bb6a6c5d85ca3b769fd8f80ed092dc98af0f --- /dev/null +++ b/experiments/breast_cancer/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "breast_cancer", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/breast_cancer/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 1, + 3, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 28, + 30, + 31, + 32, + 34, + 35, + 36, + 37, + 39 + ] +} \ No newline at end of file diff --git a/experiments/california_housing/stage4/none_with_params.json b/experiments/california_housing/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec2dc4afa018148933b0d598e28b8910476749b --- /dev/null +++ b/experiments/california_housing/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "california_housing", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/california_housing/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/california_housing/stage4/omp_with_params.json b/experiments/california_housing/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..3832ab5d326aae53010b25a75567f77ea7e12e32 --- /dev/null +++ b/experiments/california_housing/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "california_housing", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/california_housing/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/california_housing/stage4/random_with_params.json b/experiments/california_housing/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..382ecbe95e0918ef29d35bae1cf1a65d145202c7 --- /dev/null +++ b/experiments/california_housing/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "california_housing", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/california_housing/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/diabetes/stage4/none_with_params.json b/experiments/diabetes/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..e21bf5f2963127fa3d3bba9bc18364570554afc6 --- /dev/null +++ b/experiments/diabetes/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diabetes", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diabetes/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 1, + 3, + 4, + 6, + 7, + 8, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 20, + 21, + 22, + 24, + 25, + 26, + 28, + 29, + 31, + 32, + 33, + 35, + 36, + 38, + 39, + 40, + 42 + ] +} \ No newline at end of file diff --git a/experiments/diabetes/stage4/omp_with_params.json b/experiments/diabetes/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..299ea3a7b993c1f7d930e226602d4762f99dc76c --- /dev/null +++ b/experiments/diabetes/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diabetes", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diabetes/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 1, + 3, + 4, + 6, + 7, + 8, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 20, + 21, + 22, + 24, + 25, + 26, + 28, + 29, + 31, + 32, + 33, + 35, + 36, + 38, + 39, + 40, + 42 + ] +} \ No newline at end of file diff --git a/experiments/diabetes/stage4/random_with_params.json b/experiments/diabetes/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..df530656d5ae13c728d682fc037221a22fda5e98 --- /dev/null +++ b/experiments/diabetes/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diabetes", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diabetes/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 1, + 3, + 4, + 6, + 7, + 8, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 20, + 21, + 22, + 24, + 25, + 26, + 28, + 29, + 31, + 32, + 33, + 35, + 36, + 38, + 39, + 40, + 42 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage1/none_with_params.json b/experiments/diamonds/stage1/none_with_params.json index 2e8c196daac38033f035e90d2dd926c0f1833536..6ce31b8fa48688f22e5c139dddc94c61a6865ed5 100644 --- a/experiments/diamonds/stage1/none_with_params.json +++ b/experiments/diamonds/stage1/none_with_params.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "none", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage1/none_wo_params.json b/experiments/diamonds/stage1/none_wo_params.json index 626dbd59b46e591e5acca9fc8624f614528ba0b6..927e4c232bbb74756f321f6009cb8e7e029aa8d3 100644 --- a/experiments/diamonds/stage1/none_wo_params.json +++ b/experiments/diamonds/stage1/none_wo_params.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "none", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage1/omp_with_params.json b/experiments/diamonds/stage1/omp_with_params.json index a8a36be415dbed64ac21ab130e15675394de432a..e972f88e180f5ec1b9a1d6296caa9e68ac8effff 100644 --- a/experiments/diamonds/stage1/omp_with_params.json +++ b/experiments/diamonds/stage1/omp_with_params.json @@ -6,17 +6,13 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, "random_seed_number": 1, "seeds": [ - 1, - 2, - 3, - 4, 5 ], "subsets_used": "train,dev", @@ -30,9 +26,15 @@ "job_number": -1, "extraction_strategy": "omp", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage1/omp_wo_params.json b/experiments/diamonds/stage1/omp_wo_params.json index 627b9196485c29e3948aa4d39bca9598190d6b18..1989239a76ec7f9696f63e9d7fe492eb5ba58c05 100644 --- a/experiments/diamonds/stage1/omp_wo_params.json +++ b/experiments/diamonds/stage1/omp_wo_params.json @@ -5,9 +5,9 @@ "dataset_name": "diamonds", "normalize_D": false, "dataset_normalizer": "standard", - "forest_size": 1000, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "forest_size": null, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, @@ -15,10 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5, - 6 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -31,10 +28,15 @@ "job_number": -1, "extraction_strategy": "omp", "extracted_forest_size": [ - 8, - 17, - 25, - 33, - 42 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage1/random_with_params.json b/experiments/diamonds/stage1/random_with_params.json index 88a2051187e33552cd75f0906514c6a744f75c9f..b4b2ceebd371c0f361f2ce21b2917eb9c962545a 100644 --- a/experiments/diamonds/stage1/random_with_params.json +++ b/experiments/diamonds/stage1/random_with_params.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "random", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage1/random_wo_params.json b/experiments/diamonds/stage1/random_wo_params.json index 296acffdde3bd726b18f0abc43e356ab42bfe2e6..a835f689c3c28e72afdd278861c56f1e1b15fccf 100644 --- a/experiments/diamonds/stage1/random_wo_params.json +++ b/experiments/diamonds/stage1/random_wo_params.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage1", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "random", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage2/no_normalization.json b/experiments/diamonds/stage2/no_normalization.json new file mode 100644 index 0000000000000000000000000000000000000000..2af65eff187d873bed689cb408f7f3cbd60ddca8 --- /dev/null +++ b/experiments/diamonds/stage2/no_normalization.json @@ -0,0 +1,42 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage2", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train,dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "2", + "no_normalization" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage2/normalize_D.json b/experiments/diamonds/stage2/normalize_D.json new file mode 100644 index 0000000000000000000000000000000000000000..b69f4a0a59a138be13d6b5f861ed222bab310b4e --- /dev/null +++ b/experiments/diamonds/stage2/normalize_D.json @@ -0,0 +1,42 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage2", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train,dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "2", + "normalize_D" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage2/normalize_D_and_weights.json b/experiments/diamonds/stage2/normalize_D_and_weights.json new file mode 100644 index 0000000000000000000000000000000000000000..a9ca586c811faaf2b203148f7808ac544f5e7e3a --- /dev/null +++ b/experiments/diamonds/stage2/normalize_D_and_weights.json @@ -0,0 +1,42 @@ +{ + "experiment_id": 4, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": true, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage2", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train,dev", + "normalize_weights": true, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "2", + "normalize_D_and_weights" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage2/normalize_weights.json b/experiments/diamonds/stage2/normalize_weights.json new file mode 100644 index 0000000000000000000000000000000000000000..795b1a6ff650e38a4916eceb1eadb41501a7ed94 --- /dev/null +++ b/experiments/diamonds/stage2/normalize_weights.json @@ -0,0 +1,42 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage2", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train,dev", + "normalize_weights": true, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "2", + "normalize_weights" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage3/train-dev_subset.json b/experiments/diamonds/stage3/train-dev_subset.json index dd1e9493e8cee8186f704bc6c4d8edbc6d245b17..935c5063e9555b1a1dca6e8fd8bddccb43dbf866 100644 --- a/experiments/diamonds/stage3/train-dev_subset.json +++ b/experiments/diamonds/stage3/train-dev_subset.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage3", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "omp", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage3/train-dev_train-dev_subset.json b/experiments/diamonds/stage3/train-dev_train-dev_subset.json index 9a88eaa81811af3c9344c87d61591d8135a45900..9297d9c49aa08f0738a09b88595f1dae9884a0ab 100644 --- a/experiments/diamonds/stage3/train-dev_train-dev_subset.json +++ b/experiments/diamonds/stage3/train-dev_train-dev_subset.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage3", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train+dev,train+dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "omp", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage3/train-train-dev_subset.json b/experiments/diamonds/stage3/train-train-dev_subset.json index 3fca5f5b78f975ce98dd1ead8a324e1c43fdd6ed..7cdd51ba802e3efbcf3e1d709edc2bb9452f00aa 100644 --- a/experiments/diamonds/stage3/train-train-dev_subset.json +++ b/experiments/diamonds/stage3/train-train-dev_subset.json @@ -6,8 +6,8 @@ "normalize_D": false, "dataset_normalizer": "standard", "forest_size": null, - "extracted_forest_size_samples": 5, - "extracted_forest_size_stop": 0.05, + "extracted_forest_size_samples": 10, + "extracted_forest_size_stop": 0.4, "models_dir": "models/diamonds/stage3", "dev_size": 0.2, "test_size": 0.2, @@ -15,9 +15,7 @@ "seeds": [ 1, 2, - 3, - 4, - 5 + 3 ], "subsets_used": "train,train+dev", "normalize_weights": false, @@ -30,9 +28,15 @@ "job_number": -1, "extraction_strategy": "omp", "extracted_forest_size": [ - 1, - 2, - 3, - 4 + 16, + 31, + 47, + 62, + 78, + 94, + 109, + 125, + 140, + 156 ] } \ No newline at end of file diff --git a/experiments/diamonds/stage4/none_with_params.json b/experiments/diamonds/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..156b2a675fc3ffe840c7e7839010cf022e2d44f7 --- /dev/null +++ b/experiments/diamonds/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 6, + 11, + 17, + 22, + 28, + 33, + 39, + 44, + 50, + 55, + 61, + 66, + 72, + 77, + 83, + 89, + 94, + 100, + 105, + 111, + 116, + 122, + 127, + 133, + 138, + 144, + 149, + 155, + 161, + 166 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage4/omp_with_params.json b/experiments/diamonds/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..ef97c05b05ef97edb56baf39923fbbd1a13d6f97 --- /dev/null +++ b/experiments/diamonds/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 6, + 11, + 17, + 22, + 28, + 33, + 39, + 44, + 50, + 55, + 61, + 66, + 72, + 77, + 83, + 89, + 94, + 100, + 105, + 111, + 116, + 122, + 127, + 133, + 138, + 144, + 149, + 155, + 161, + 166 + ] +} \ No newline at end of file diff --git a/experiments/diamonds/stage4/random_with_params.json b/experiments/diamonds/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..2fd6858479b606873210c910f52eb94dbce89bac --- /dev/null +++ b/experiments/diamonds/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "diamonds", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/diamonds/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 6, + 11, + 17, + 22, + 28, + 33, + 39, + 44, + 50, + 55, + 61, + 66, + 72, + 77, + 83, + 89, + 94, + 100, + 105, + 111, + 116, + 122, + 127, + 133, + 138, + 144, + 149, + 155, + 161, + 166 + ] +} \ No newline at end of file diff --git a/experiments/digits/stage4/none_with_params.json b/experiments/digits/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..3645703cb07190abca4e8dfa3c914f0e279dd065 --- /dev/null +++ b/experiments/digits/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "digits", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/digits/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/digits/stage4/omp_with_params.json b/experiments/digits/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..400aed98ea1ce0839876fd6cab338d3448eec88e --- /dev/null +++ b/experiments/digits/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "digits", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/digits/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/digits/stage4/random_with_params.json b/experiments/digits/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..ea5c8e4c99979407f6a85fca44ded05527396c5c --- /dev/null +++ b/experiments/digits/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "digits", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/digits/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/iris/stage4/none_with_params.json b/experiments/iris/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..711fd2f5b2e19aec02b72f0cab28543c4023167f --- /dev/null +++ b/experiments/iris/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "iris", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/iris/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/iris/stage4/omp_with_params.json b/experiments/iris/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..e45084843b1b627ab28587e34591e78df7399290 --- /dev/null +++ b/experiments/iris/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "iris", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/iris/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/iris/stage4/random_with_params.json b/experiments/iris/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..06c8c9e9ef0f92a6e4946e16823ccd59bf3ce26a --- /dev/null +++ b/experiments/iris/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "iris", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/iris/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/lfw_pairs/stage4/none_with_params.json b/experiments/lfw_pairs/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..d2c8bdf0129663490270dea33234b50c34e60cda --- /dev/null +++ b/experiments/lfw_pairs/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "lfw_pairs", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/lfw_pairs/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/lfw_pairs/stage4/omp_with_params.json b/experiments/lfw_pairs/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..2368bc2f0157dcfc9875cac38d95e07accd91945 --- /dev/null +++ b/experiments/lfw_pairs/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "lfw_pairs", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/lfw_pairs/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/lfw_pairs/stage4/random_with_params.json b/experiments/lfw_pairs/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..b4cbce9a023354ec931aa06665b5c9a06f5523cc --- /dev/null +++ b/experiments/lfw_pairs/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "lfw_pairs", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/lfw_pairs/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/linnerud/stage4/none_with_params.json b/experiments/linnerud/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..dcb15acd294aae4b3fc81e02dcbeef6d777254cf --- /dev/null +++ b/experiments/linnerud/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "linnerud", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/linnerud/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/linnerud/stage4/omp_with_params.json b/experiments/linnerud/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..a5014456354a3b2efbb1c8e50183f74e9f6d2856 --- /dev/null +++ b/experiments/linnerud/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "linnerud", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/linnerud/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/linnerud/stage4/random_with_params.json b/experiments/linnerud/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..cf59ff4656803e9ec3223176be1ecff45fa309d1 --- /dev/null +++ b/experiments/linnerud/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "linnerud", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/linnerud/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/olivetti_faces/stage4/none_with_params.json b/experiments/olivetti_faces/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..eb46dd826e4c040bbb0b1233f6b3969c3515bf3e --- /dev/null +++ b/experiments/olivetti_faces/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "olivetti_faces", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/olivetti_faces/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/olivetti_faces/stage4/omp_with_params.json b/experiments/olivetti_faces/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..2f73580d73d9353b6d0ef7b0f251a345c007980a --- /dev/null +++ b/experiments/olivetti_faces/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "olivetti_faces", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/olivetti_faces/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/olivetti_faces/stage4/random_with_params.json b/experiments/olivetti_faces/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..101090eacba707ad7eb7a5006361616a2b7baccc --- /dev/null +++ b/experiments/olivetti_faces/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "olivetti_faces", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/olivetti_faces/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/wine/stage4/none_with_params.json b/experiments/wine/stage4/none_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..015200fa33983172c72091a8de2b922130951b75 --- /dev/null +++ b/experiments/wine/stage4/none_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 1, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "wine", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/wine/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "none_with_params" + ], + "job_number": -1, + "extraction_strategy": "none", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/wine/stage4/omp_with_params.json b/experiments/wine/stage4/omp_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..9289c0d7e688e4d4bc507c8627a97b080b75ba21 --- /dev/null +++ b/experiments/wine/stage4/omp_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 3, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "wine", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/wine/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "omp_with_params" + ], + "job_number": -1, + "extraction_strategy": "omp", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/experiments/wine/stage4/random_with_params.json b/experiments/wine/stage4/random_with_params.json new file mode 100644 index 0000000000000000000000000000000000000000..a0cc59b6578270e59ec45f7dfd3b9bc36647d004 --- /dev/null +++ b/experiments/wine/stage4/random_with_params.json @@ -0,0 +1,62 @@ +{ + "experiment_id": 2, + "experiment_configuration": null, + "experiment_configuration_path": "experiments", + "dataset_name": "wine", + "normalize_D": false, + "dataset_normalizer": "standard", + "forest_size": null, + "extracted_forest_size_samples": 30, + "extracted_forest_size_stop": 0.4, + "models_dir": "models/wine/stage4", + "dev_size": 0.2, + "test_size": 0.2, + "random_seed_number": 1, + "seeds": [ + 1, + 2, + 3 + ], + "subsets_used": "train+dev,train+dev", + "normalize_weights": false, + "verbose": false, + "skip_best_hyperparams": false, + "save_experiment_configuration": [ + "4", + "random_with_params" + ], + "job_number": -1, + "extraction_strategy": "random", + "extracted_forest_size": [ + 13, + 26, + 39, + 52, + 65, + 77, + 90, + 103, + 116, + 129, + 142, + 155, + 168, + 181, + 194, + 206, + 219, + 232, + 245, + 258, + 271, + 284, + 297, + 310, + 323, + 335, + 348, + 361, + 374, + 387 + ] +} \ No newline at end of file diff --git a/results/boston/stage4/losses.png b/results/boston/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..ab2f45d2f8744176f158dd9d070c9dba7d8100a6 Binary files /dev/null and b/results/boston/stage4/losses.png differ diff --git a/results/breast_cancer/stage4/losses.png b/results/breast_cancer/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..44268f88352ea22c7a326dcffc71e71070ac7274 Binary files /dev/null and b/results/breast_cancer/stage4/losses.png differ diff --git a/results/california_housing/stage4/losses.png b/results/california_housing/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..fc285aad17f21b80886457d5915fdf723309484e Binary files /dev/null and b/results/california_housing/stage4/losses.png differ diff --git a/results/diabetes/stage4/losses.png b/results/diabetes/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..f2fbccddb99359a9b12250ce339765f4e4946131 Binary files /dev/null and b/results/diabetes/stage4/losses.png differ diff --git a/results/diamonds/stage1/losses.png b/results/diamonds/stage1/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..3a28f702a447f518de9dfc0c85639c98f6d0ba96 Binary files /dev/null and b/results/diamonds/stage1/losses.png differ diff --git a/results/diamonds/stage2/losses.png b/results/diamonds/stage2/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..eba0647c981e203e1b305a0a2daf30a5b9037bed Binary files /dev/null and b/results/diamonds/stage2/losses.png differ diff --git a/results/diamonds/stage3/losses.png b/results/diamonds/stage3/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..0c05e27cd2fe32083ed2706aec5ccd3a738d1e77 Binary files /dev/null and b/results/diamonds/stage3/losses.png differ diff --git a/results/diamonds/stage4/losses.png b/results/diamonds/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..d6da7b1cb6f9d17de1dff72af5dc11f124a310c2 Binary files /dev/null and b/results/diamonds/stage4/losses.png differ diff --git a/results/digits/stage4/losses.png b/results/digits/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..cabee4be51b813c38a66a1d478f3d3fa7f2a3181 Binary files /dev/null and b/results/digits/stage4/losses.png differ diff --git a/results/iris/stage4/losses.png b/results/iris/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..f9a57ccb6de7c466babbe1c78ec29a429cb50a9d Binary files /dev/null and b/results/iris/stage4/losses.png differ diff --git a/results/olivetti_faces/stage4/losses.png b/results/olivetti_faces/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..0dfb13b327925ab329e1e3b014d65a880fe596d0 Binary files /dev/null and b/results/olivetti_faces/stage4/losses.png differ diff --git a/results/wine/stage4/losses.png b/results/wine/stage4/losses.png new file mode 100644 index 0000000000000000000000000000000000000000..2f71e9a25f311da1014cb2148c6d88cbb616fa64 Binary files /dev/null and b/results/wine/stage4/losses.png differ diff --git a/scripts/run_compute_results.sh b/scripts/run_compute_results.sh index 7f1e22d0134a5048c32df143949d9b5306da4ee4..f9f130e19c4d467e9d0416a051b8353f071b42dd 100644 --- a/scripts/run_compute_results.sh +++ b/scripts/run_compute_results.sh @@ -1,6 +1,7 @@ -for dataset in california_housing boston iris diabetes digits linnerud wine breast_cancer olivetti_faces 20newsgroups_vectorized lfw_pairs diamonds +for dataset in diamonds california_housing boston iris diabetes digits linnerud wine breast_cancer olivetti_faces 20newsgroups_vectorized lfw_pairs do python code/compute_results.py --stage=1 --experiment_ids 1 2 3 4 5 6 --dataset_name=$dataset --models_dir=models/$dataset/stage1 python code/compute_results.py --stage=2 --experiment_ids 1 2 3 4 --dataset_name=$dataset --models_dir=models/$dataset/stage2 python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=$dataset --models_dir=models/$dataset/stage3 + python code/compute_results.py --stage=4 --experiment_ids 1 2 3 --dataset_name=$dataset --models_dir=models/$dataset/stage4 done diff --git a/scripts/run_compute_results_fix.sh b/scripts/run_compute_results_fix.sh new file mode 100644 index 0000000000000000000000000000000000000000..a2a3f9c0205aa1b6c3162a4a6a5b5ac9bbcd7874 --- /dev/null +++ b/scripts/run_compute_results_fix.sh @@ -0,0 +1,11 @@ +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=california_housing --models_dir=models/california_housing/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=boston --models_dir=models/boston/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=iris --models_dir=models/iris/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=diabetes --models_dir=models/diabetes/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=digits --models_dir=models/digits/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=linnerud --models_dir=models/linnerud/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=wine --models_dir=models/wine/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=breast_cancer --models_dir=models/breast_cancer/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=olivetti_faces --models_dir=models/olivetti_faces/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=20newsgroups_vectorized --models_dir=models/20newsgroups_vectorized/stage3 +python code/compute_results.py --stage=3 --experiment_ids 1 2 3 --dataset_name=lfw_pairs --models_dir=models/lfw_pairs/stage3 \ No newline at end of file diff --git a/scripts/run_stage1_experiments_fix.sh b/scripts/run_stage1_experiments_fix.sh new file mode 100644 index 0000000000000000000000000000000000000000..7ffa5c770300a774e584648fa2a671b493fa15fd --- /dev/null +++ b/scripts/run_stage1_experiments_fix.sh @@ -0,0 +1,14 @@ +#!/bin/bash +core_number=10 +walltime=1:00 +seeds='1 2 3' + +for dataset in diamonds +do + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=none --save_experiment_configuration 1 none_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=1 --models_dir=models/$dataset/stage1" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=random --save_experiment_configuration 1 random_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=2 --models_dir=models/$dataset/stage1" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=5:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds 5 --save_experiment_configuration 1 omp_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=3 --models_dir=models/$dataset/stage1" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=none --skip_best_hyperparams --save_experiment_configuration 1 none_wo_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=4 --models_dir=models/$dataset/stage1" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=random --skip_best_hyperparams --save_experiment_configuration 1 random_wo_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=5 --models_dir=models/$dataset/stage1" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=5:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --skip_best_hyperparams --save_experiment_configuration 1 omp_wo_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=6 --models_dir=models/$dataset/stage1" +done diff --git a/scripts/run_stage2_experiments_fix.sh b/scripts/run_stage2_experiments_fix.sh new file mode 100644 index 0000000000000000000000000000000000000000..908112c6b65d2cb825250bb42d0985744825ad78 --- /dev/null +++ b/scripts/run_stage2_experiments_fix.sh @@ -0,0 +1,12 @@ +#!/bin/bash +core_number=10 +walltime=1:00 +seeds='1 2 3' + +for dataset in diamonds +do + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 2 no_normalization --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=1 --models_dir=models/$dataset/stage2" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 2 normalize_D --normalize_D --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=2 --models_dir=models/$dataset/stage2" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 2 normalize_weights --normalize_weights --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=3 --models_dir=models/$dataset/stage2" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 2 normalize_D_and_weights --normalize_D --normalize_weights --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --experiment_id=4 --models_dir=models/$dataset/stage2" +done diff --git a/scripts/run_stage3_experiments_fix.sh b/scripts/run_stage3_experiments_fix.sh new file mode 100644 index 0000000000000000000000000000000000000000..b672202b4eafd5f3123b72990efc4db3081ce3a7 --- /dev/null +++ b/scripts/run_stage3_experiments_fix.sh @@ -0,0 +1,11 @@ +#!/bin/bash +core_number=10 +walltime=1:00 +seeds='1 2 3' + +for dataset in diamonds +do + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 3 train-dev_subset --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --subsets_used=train,dev --experiment_id=1 --models_dir=models/$dataset/stage3" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 3 train-dev_train-dev_subset --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --subsets_used=train+dev,train+dev --experiment_id=2 --models_dir=models/$dataset/stage3" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --save_experiment_configuration 3 train-train-dev_subset --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=10 --subsets_used=train,train+dev --experiment_id=3 --models_dir=models/$dataset/stage3" +done diff --git a/scripts/run_stage4_experiments.sh b/scripts/run_stage4_experiments.sh new file mode 100644 index 0000000000000000000000000000000000000000..9e0043a6be24bd8169eff55acf63c2531595a5fc --- /dev/null +++ b/scripts/run_stage4_experiments.sh @@ -0,0 +1,11 @@ +#!/bin/bash +core_number=5 +walltime=1:00 +seeds='1 2 3' + +for dataset in diamonds california_housing boston iris diabetes digits linnerud wine breast_cancer olivetti_faces 20newsgroups_vectorized lfw_pairs +do + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=none --save_experiment_configuration 4 none_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=30 --experiment_id=1 --models_dir=models/$dataset/stage4 --subsets_used train+dev,train+dev" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=random --save_experiment_configuration 4 random_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=30 --experiment_id=2 --models_dir=models/$dataset/stage4 --subsets_used train+dev,train+dev" + oarsub -p "(gpu is null)" -l /core=$core_number,walltime=1:00 "conda activate test_env && python code/train.py --dataset_name=$dataset --seeds $seeds --extraction_strategy=omp --save_experiment_configuration 4 omp_with_params --extracted_forest_size_stop=0.40 --extracted_forest_size_samples=30 --experiment_id=3 --models_dir=models/$dataset/stage4 --subsets_used train+dev,train+dev" +done