compute_results.py 28.1 KB
Newer Older
1
from bolsonaro.models.model_raw_results import ModelRawResults
2
from bolsonaro.visualization.plotter import Plotter
3
4
from bolsonaro import LOG_PATH
from bolsonaro.error_handling.logger_factory import LoggerFactory
5

6
7
import argparse
import pathlib
8
9
from dotenv import find_dotenv, load_dotenv
import os
10
11


12
13
14
15
16
17
18
19
def retreive_extracted_forest_sizes_number(models_dir, experiment_id):
    experiment_id_path = models_dir + os.sep + str(experiment_id) # models/{experiment_id}
    experiment_seed_root_path = experiment_id_path + os.sep + 'seeds' # models/{experiment_id}/seeds
    seed = os.listdir(experiment_seed_root_path)[0]
    experiment_seed_path = experiment_seed_root_path + os.sep + seed # models/{experiment_id}/seeds/{seed}
    extracted_forest_sizes_root_path = experiment_seed_path + os.sep + 'extracted_forest_sizes'
    return len(os.listdir(extracted_forest_sizes_root_path))

Léo Bouscarrat's avatar
Léo Bouscarrat committed
20
def extract_scores_across_seeds_and_extracted_forest_sizes(models_dir, results_dir, experiment_id, weights=True):
21
22
23
24
25
26
27
28
29
30
    experiment_id_path = models_dir + os.sep + str(experiment_id) # models/{experiment_id}
    experiment_seed_root_path = experiment_id_path + os.sep + 'seeds' # models/{experiment_id}/seeds

    """
    Dictionaries to temporarly store the scalar results with the following structure:
    {seed_1: [score_1, ..., score_m], ... seed_n: [score_1, ..., score_k]}
    """
    experiment_train_scores = dict()
    experiment_dev_scores = dict()
    experiment_test_scores = dict()
31
    experiment_weights = dict()
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
    all_extracted_forest_sizes = list()

    # Used to check if all losses were computed using the same metric (it should be the case)
    experiment_score_metrics = list()

    # For each seed results stored in models/{experiment_id}/seeds
    seeds = os.listdir(experiment_seed_root_path)
    seeds.sort(key=int)
    for seed in seeds:
        experiment_seed_path = experiment_seed_root_path + os.sep + seed # models/{experiment_id}/seeds/{seed}
        extracted_forest_sizes_root_path = experiment_seed_path + os.sep + 'extracted_forest_sizes' # models/{experiment_id}/seeds/{seed}/forest_size

        # {{seed}:[]}
        experiment_train_scores[seed] = list()
        experiment_dev_scores[seed] = list()
        experiment_test_scores[seed] = list()
48
        experiment_weights[seed] = list()
49
50
51

        # List the forest sizes in models/{experiment_id}/seeds/{seed}/extracted_forest_sizes
        extracted_forest_sizes = os.listdir(extracted_forest_sizes_root_path)
Léo Bouscarrat's avatar
Léo Bouscarrat committed
52
        extracted_forest_sizes = [nb_tree for nb_tree in extracted_forest_sizes if not 'no_weights' in nb_tree ]
53
54
55
56
        extracted_forest_sizes.sort(key=int)
        all_extracted_forest_sizes.append(list(map(int, extracted_forest_sizes)))
        for extracted_forest_size in extracted_forest_sizes:
            # models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}
Léo Bouscarrat's avatar
Léo Bouscarrat committed
57
58
59
60
            if weights:
                extracted_forest_size_path = extracted_forest_sizes_root_path + os.sep + extracted_forest_size
            else:
                extracted_forest_size_path = extracted_forest_sizes_root_path + os.sep + extracted_forest_size + '_no_weights'
61
62
63
64
65
66
67
68
            # Load models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}/model_raw_results.pickle file
            model_raw_results = ModelRawResults.load(extracted_forest_size_path)
            # Save the scores
            experiment_train_scores[seed].append(model_raw_results.train_score)
            experiment_dev_scores[seed].append(model_raw_results.dev_score)
            experiment_test_scores[seed].append(model_raw_results.test_score)
            # Save the metric
            experiment_score_metrics.append(model_raw_results.score_metric)
69
70
            # Save the weights
            #experiment_weights[seed].append(model_raw_results.model_weights)
71
72
73
74
75
76
77

    # Sanity checks
    if len(set(experiment_score_metrics)) > 1:
        raise ValueError("The metrics used to compute the scores aren't the sames across seeds.")
    if len(set([sum(extracted_forest_sizes) for extracted_forest_sizes in all_extracted_forest_sizes])) != 1:
        raise ValueError("The extracted forest sizes aren't the sames across seeds.")

78
79
    return experiment_train_scores, experiment_dev_scores, experiment_test_scores, \
        all_extracted_forest_sizes[0], experiment_score_metrics[0]#, experiment_weights
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

def extract_scores_across_seeds_and_forest_size(models_dir, results_dir, experiment_id, extracted_forest_sizes_number):
    experiment_id_path = models_dir + os.sep + str(experiment_id) # models/{experiment_id}
    experiment_seed_root_path = experiment_id_path + os.sep + 'seeds' # models/{experiment_id}/seeds

    """
    Dictionaries to temporarly store the scalar results with the following structure:
    {seed_1: [score_1, ..., score_m], ... seed_n: [score_1, ..., score_k]}
    """
    experiment_train_scores = dict()
    experiment_dev_scores = dict()
    experiment_test_scores = dict()

    # Used to check if all losses were computed using the same metric (it should be the case)
    experiment_score_metrics = list()

    # For each seed results stored in models/{experiment_id}/seeds
    seeds = os.listdir(experiment_seed_root_path)
    seeds.sort(key=int)
    for seed in seeds:
        experiment_seed_path = experiment_seed_root_path + os.sep + seed # models/{experiment_id}/seeds/{seed}
        forest_size_root_path = experiment_seed_path + os.sep + 'forest_size' # models/{experiment_id}/seeds/{seed}/forest_size

        # {{seed}:[]}
        experiment_train_scores[seed] = list()
        experiment_dev_scores[seed] = list()
        experiment_test_scores[seed] = list()

        forest_size = os.listdir(forest_size_root_path)[0]
        # models/{experiment_id}/seeds/{seed}/forest_size/{forest_size}
        forest_size_path = forest_size_root_path + os.sep + forest_size
        # Load models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}/model_raw_results.pickle file
        model_raw_results = ModelRawResults.load(forest_size_path)
        for _ in range(extracted_forest_sizes_number):
            # Save the scores
            experiment_train_scores[seed].append(model_raw_results.train_score)
            experiment_dev_scores[seed].append(model_raw_results.dev_score)
            experiment_test_scores[seed].append(model_raw_results.test_score)
            # Save the metric
            experiment_score_metrics.append(model_raw_results.score_metric)

    if len(set(experiment_score_metrics)) > 1:
        raise ValueError("The metrics used to compute the scores aren't the same everytime")

124
    return experiment_train_scores, experiment_dev_scores, experiment_test_scores, experiment_score_metrics[0]
125

126
if __name__ == "__main__":
127
    # get environment variables in .env
Charly Lamothe's avatar
Charly Lamothe committed
128
    load_dotenv(find_dotenv('.env'))
129

Charly LAMOTHE's avatar
Charly LAMOTHE committed
130
131
    DEFAULT_RESULTS_DIR = os.environ["project_dir"] + os.sep + 'results'
    DEFAULT_MODELS_DIR = os.environ["project_dir"] + os.sep + 'models'
132
    DEFAULT_PLOT_WEIGHT_DENSITY = False
133
134

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
Charly Lamothe's avatar
Charly Lamothe committed
135
    parser.add_argument('--stage', nargs='?', type=int, required=True, help='Specify the stage number among [1, 5].')
136
    parser.add_argument('--experiment_ids', nargs='+', type=str, required=True, help='Compute the results of the specified experiment id(s).' + \
137
138
        'stage=1: {{base_with_params}} {{random_with_params}} {{omp_with_params}} {{base_wo_params}} {{random_wo_params}} {{omp_wo_params}}' + \
        'stage=2: {{no_normalization}} {{normalize_D}} {{normalize_weights}} {{normalize_D_and_weights}}' + \
139
140
        'stage=3: {{train-dev_subset}} {{train-dev_train-dev_subset}} {{train-train-dev_subset}}' + \
        'stage=5: {{base_with_params}} {{random_with_params}} {{omp_with_params}} [ensemble={{id}}] [similarity={{id}}] [kmean={{id}}]')
141
    parser.add_argument('--dataset_name', nargs='?', type=str, required=True, help='Specify the dataset name. TODO: read it from models dir directly.')
Charly LAMOTHE's avatar
Charly LAMOTHE committed
142
143
    parser.add_argument('--results_dir', nargs='?', type=str, default=DEFAULT_RESULTS_DIR, help='The output directory of the results.')
    parser.add_argument('--models_dir', nargs='?', type=str, default=DEFAULT_MODELS_DIR, help='The output directory of the trained models.')
144
    parser.add_argument('--plot_weight_density', action='store_true', default=DEFAULT_PLOT_WEIGHT_DENSITY, help='Plot the weight density. Only working for regressor models for now.')
145
146
    args = parser.parse_args()

Charly Lamothe's avatar
Charly Lamothe committed
147
148
    if args.stage not in list(range(1, 6)):
        raise ValueError('stage must be a supported stage id (i.e. [1, 5]).')
149

150
151
    logger = LoggerFactory.create(LOG_PATH, os.path.basename(__file__))

152
153
154
    logger.info('Compute results of with stage:{} - experiment_ids:{} - dataset_name:{} - results_dir:{} - models_dir:{}'.format(
        args.stage, args.experiment_ids, args.dataset_name, args.results_dir, args.models_dir))

155
    # Create recursively the results dir tree
156
157
    pathlib.Path(args.results_dir).mkdir(parents=True, exist_ok=True)

158
    if args.stage == 1:
159
160
161
        if len(args.experiment_ids) != 6:
            raise ValueError('In the case of stage 1, the number of specified experiment ids must be 6.')

162
        # Retreive the extracted forest sizes number used in order to have a base forest axis as long as necessary
163
        extracted_forest_sizes_number = retreive_extracted_forest_sizes_number(args.models_dir, int(args.experiment_ids[1]))
164

165
166
        # Experiments that used the best hyperparameters found for this dataset

167
        # base_with_params
168
        logger.info('Loading base_with_params experiment scores...')
169
170
        base_with_params_train_scores, base_with_params_dev_scores, base_with_params_test_scores, \
            base_with_params_experiment_score_metric = \
171
            extract_scores_across_seeds_and_forest_size(args.models_dir, args.results_dir, int(args.experiment_ids[0]),
172
            extracted_forest_sizes_number)
173
        # random_with_params
174
175
        logger.info('Loading random_with_params experiment scores...')
        random_with_params_train_scores, random_with_params_dev_scores, random_with_params_test_scores, \
176
            with_params_extracted_forest_sizes, random_with_params_experiment_score_metric = \
177
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, int(args.experiment_ids[1]))
178
        # omp_with_params
179
        logger.info('Loading omp_with_params experiment scores...')
180
181
        omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores, _, \
            omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
182
                args.models_dir, args.results_dir, int(args.experiment_ids[2]))
183
184

        # Experiments that didn't use the best hyperparameters found for this dataset
185
186

        # base_wo_params
187
        logger.info('Loading base_wo_params experiment scores...')
188
189
        base_wo_params_train_scores, base_wo_params_dev_scores, base_wo_params_test_scores, \
            base_wo_params_experiment_score_metric = extract_scores_across_seeds_and_forest_size(
190
                args.models_dir, args.results_dir, int(args.experiment_ids[3]),
191
            extracted_forest_sizes_number)
192
        # random_wo_params
193
194
        logger.info('Loading random_wo_params experiment scores...')
        random_wo_params_train_scores, random_wo_params_dev_scores, random_wo_params_test_scores, \
195
196
            wo_params_extracted_forest_sizes, random_wo_params_experiment_score_metric = \
                extract_scores_across_seeds_and_extracted_forest_sizes(
197
                args.models_dir, args.results_dir, int(args.experiment_ids[4]))
198
        # base_wo_params
199
        logger.info('Loading base_wo_params experiment scores...')
200
201
        omp_wo_params_train_scores, omp_wo_params_dev_scores, omp_wo_params_test_scores, _, \
            omp_wo_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
202
                args.models_dir, args.results_dir, int(args.experiment_ids[5]))
203
204
205
206
207
208
209
210

        # Sanity check on the metrics retreived
        if not (base_with_params_experiment_score_metric == random_with_params_experiment_score_metric ==
            omp_with_params_experiment_score_metric == base_wo_params_experiment_score_metric ==
            random_wo_params_experiment_score_metric ==
            omp_wo_params_experiment_score_metric):
            raise ValueError('Score metrics of all experiments must be the same.')
        experiments_score_metric = base_with_params_experiment_score_metric
211
212
213
214

        output_path = os.path.join(args.results_dir, args.dataset_name, 'stage1')
        pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)

215
        """all_experiment_scores_with_params=[base_with_params_train_scores, base_with_params_dev_scores, base_with_params_test_scores,
216
217
                random_with_params_train_scores, random_with_params_dev_scores, random_with_params_test_scores,
                omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores],
218
            all_experiment_scores_wo_params=[base_wo_params_train_scores, base_wo_params_dev_scores, base_wo_params_test_scores,
219
220
                random_wo_params_train_scores, random_wo_params_dev_scores, random_wo_params_test_scores,
                omp_wo_params_train_scores, omp_wo_params_dev_scores, omp_wo_params_test_scores],
221
222
223
224
225
226
227
228
229
230
231
232
233
            all_labels=['base_with_params_train', 'base_with_params_dev', 'base_with_params_test',
                'random_with_params_train', 'random_with_params_dev', 'random_with_params_test',
                'omp_with_params_train', 'omp_with_params_dev', 'omp_with_params_test'],"""

        Plotter.plot_stage1_losses(
            file_path=output_path + os.sep + 'losses.png',
            all_experiment_scores_with_params=[base_with_params_test_scores,
                random_with_params_test_scores,
                omp_with_params_test_scores],
            all_experiment_scores_wo_params=[base_wo_params_test_scores,
                random_wo_params_test_scores,
                omp_wo_params_test_scores],
            all_labels=['base', 'random', 'omp'],
234
235
            x_value=with_params_extracted_forest_sizes,
            xlabel='Number of trees extracted',
236
            ylabel=experiments_score_metric,
237
            title='Loss values of {}\nusing best and default hyperparameters'.format(args.dataset_name)
238
        )
239
240
241
242
243
244
245
246
    elif args.stage == 2:
        if len(args.experiment_ids) != 4:
            raise ValueError('In the case of stage 2, the number of specified experiment ids must be 4.')

        # no_normalization
        logger.info('Loading no_normalization experiment scores...')
        _, _, no_normalization_test_scores, extracted_forest_sizes, no_normalization_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
247
            int(args.experiment_ids[0]))
248
249
250
251
252

        # normalize_D
        logger.info('Loading normalize_D experiment scores...')
        _, _, normalize_D_test_scores, _, normalize_D_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
253
            int(args.experiment_ids[1]))
254
255
256
257
258

        # normalize_weights
        logger.info('Loading normalize_weights experiment scores...')
        _, _, normalize_weights_test_scores, _, normalize_weights_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
259
            int(args.experiment_ids[2]))
260
261
262
263
264

        # normalize_D_and_weights
        logger.info('Loading normalize_D_and_weights experiment scores...')
        _, _, normalize_D_and_weights_test_scores, _, normalize_D_and_weights_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
265
            int(args.experiment_ids[3]))
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293

        # Sanity check on the metrics retreived
        if not (no_normalization_experiment_score_metric == normalize_D_experiment_score_metric
            == normalize_weights_experiment_score_metric == normalize_D_and_weights_experiment_score_metric):
            raise ValueError('Score metrics of all experiments must be the same.')
        experiments_score_metric = no_normalization_experiment_score_metric

        output_path = os.path.join(args.results_dir, args.dataset_name, 'stage2')
        pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)

        Plotter.plot_stage2_losses(
            file_path=output_path + os.sep + 'losses.png',
            all_experiment_scores=[no_normalization_test_scores, normalize_D_test_scores,
                normalize_weights_test_scores, normalize_D_and_weights_test_scores],
            all_labels=['no_normalization', 'normalize_D', 'normalize_weights', 'normalize_D_and_weights'],
            x_value=extracted_forest_sizes,
            xlabel='Number of trees extracted',
            ylabel=experiments_score_metric,
            title='Loss values of {}\nusing different normalizations'.format(args.dataset_name))
    elif args.stage == 3:
        if len(args.experiment_ids) != 3:
            raise ValueError('In the case of stage 3, the number of specified experiment ids must be 3.')

        # train-dev_subset
        logger.info('Loading train-dev_subset experiment scores...')
        train_dev_subset_train_scores, train_dev_subset_dev_scores, train_dev_subset_test_scores, \
            extracted_forest_sizes, train_dev_subset_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
294
            int(args.experiment_ids[0]))
295
296
297
298
299
300

        # train-dev_train-dev_subset
        logger.info('Loading train-dev_train-dev_subset experiment scores...')
        train_dev_train_dev_subset_train_scores, train_dev_train_dev_subset_dev_scores, train_dev_train_dev_subset_test_scores, \
            _, train_dev_train_dev_subset_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
301
            int(args.experiment_ids[1]))
302
303
304
305
306
307

        # train-train-dev_subset
        logger.info('Loading train-train-dev_subset experiment scores...')
        train_train_dev_subset_train_scores, train_train_dev_subset_dev_scores, train_train_dev_subset_test_scores, \
            _, train_train_dev_subset_experiment_score_metric = \
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir,
308
            int(args.experiment_ids[2]))
309
310
311
312
313
314
315
316
317
318
319

        # Sanity check on the metrics retreived
        if not (train_dev_subset_experiment_score_metric == train_dev_train_dev_subset_experiment_score_metric
            == train_train_dev_subset_experiment_score_metric):
            raise ValueError('Score metrics of all experiments must be the same.')
        experiments_score_metric = train_dev_subset_experiment_score_metric

        output_path = os.path.join(args.results_dir, args.dataset_name, 'stage3')
        pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)

        Plotter.plot_stage2_losses(
320
321
322
323
324
325
326
327
328
329
            file_path=output_path + os.sep + 'losses.png',
            all_experiment_scores=[train_dev_subset_test_scores, train_dev_train_dev_subset_test_scores,
                train_train_dev_subset_test_scores],
            all_labels=['train,dev', 'train+dev,train+dev', 'train,train+dev'],
            x_value=extracted_forest_sizes,
            xlabel='Number of trees extracted',
            ylabel=experiments_score_metric,
            title='Loss values of {}\nusing different training subsets'.format(args.dataset_name))

        """Plotter.plot_stage2_losses(
330
331
332
333
334
335
336
337
338
339
340
            file_path=output_path + os.sep + 'losses.png',
            all_experiment_scores=[train_dev_subset_train_scores, train_train_dev_subset_train_scores,
                train_train_dev_subset_train_scores, train_dev_subset_dev_scores, train_dev_train_dev_subset_dev_scores,
                train_train_dev_subset_dev_scores, train_dev_subset_test_scores, train_dev_train_dev_subset_test_scores,
                train_train_dev_subset_test_scores],
            all_labels=['train,dev - train', 'train+dev,train+dev - train', 'train,train+dev - train',
                'train,dev - dev', 'train+dev,train+dev - dev', 'train,train+dev - dev',
                'train,dev - test', 'train+dev,train+dev - test', 'train,train+dev - test'],
            x_value=extracted_forest_sizes,
            xlabel='Number of trees extracted',
            ylabel=experiments_score_metric,
341
            title='Loss values of {}\nusing different training subsets'.format(args.dataset_name))"""
Charly Lamothe's avatar
Charly Lamothe committed
342
    elif args.stage == 4:
343
        if len(args.experiment_ids) != 3:
Charly Lamothe's avatar
Charly Lamothe committed
344
345
346
347
348
349
350
351
352
            raise ValueError('In the case of stage 4, the number of specified experiment ids must be 3.')

        # Retreive the extracted forest sizes number used in order to have a base forest axis as long as necessary
        extracted_forest_sizes_number = retreive_extracted_forest_sizes_number(args.models_dir, args.experiment_ids[1])

        # base_with_params
        logger.info('Loading base_with_params experiment scores...')
        base_with_params_train_scores, base_with_params_dev_scores, base_with_params_test_scores, \
            base_with_params_experiment_score_metric = \
353
            extract_scores_across_seeds_and_forest_size(args.models_dir, args.results_dir, int(args.experiment_ids[0]),
Charly Lamothe's avatar
Charly Lamothe committed
354
355
356
357
358
            extracted_forest_sizes_number)
        # random_with_params
        logger.info('Loading random_with_params experiment scores...')
        random_with_params_train_scores, random_with_params_dev_scores, random_with_params_test_scores, \
            with_params_extracted_forest_sizes, random_with_params_experiment_score_metric = \
359
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, int(args.experiment_ids[1]))
Charly Lamothe's avatar
Charly Lamothe committed
360
361
        # omp_with_params
        logger.info('Loading omp_with_params experiment scores...')
362
363
364
        """omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores, _, \
            omp_with_params_experiment_score_metric, experiment_weights = extract_scores_across_seeds_and_extracted_forest_sizes(
                args.models_dir, args.results_dir, args.experiment_ids[2])"""
Charly Lamothe's avatar
Charly Lamothe committed
365
366
        omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores, _, \
            omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
367
                args.models_dir, args.results_dir, int(args.experiment_ids[2]))
Léo Bouscarrat's avatar
Léo Bouscarrat committed
368
369
370
371
        #omp_with_params_without_weights
        logger.info('Loading omp_with_params experiment scores...')
        omp_with_params_without_weights_train_scores, omp_with_params_without_weights_dev_scores, omp_with_params_without_weights_test_scores, _, \
            omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
372
                args.models_dir, args.results_dir, int(args.experiment_ids[2]), weights=False)
Charly Lamothe's avatar
Charly Lamothe committed
373

374
        """# base_with_params
375
376
377
378
379
380
381
382
383
        logger.info('Loading base_with_params experiment scores 2...')
        _, _, base_with_params_test_scores_2, \
            _ = \
            extract_scores_across_seeds_and_forest_size(args.models_dir, args.results_dir, args.experiment_ids[3],
            extracted_forest_sizes_number)
        # random_with_params
        logger.info('Loading random_with_params experiment scores 2...')
        _, _, random_with_params_test_scores_2, \
            _, _ = \
384
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, args.experiment_ids[4])"""
385

Charly Lamothe's avatar
Charly Lamothe committed
386
387
388
389
390
391
        # Sanity check on the metrics retreived
        if not (base_with_params_experiment_score_metric == random_with_params_experiment_score_metric
            == omp_with_params_experiment_score_metric):
            raise ValueError('Score metrics of all experiments must be the same.')
        experiments_score_metric = base_with_params_experiment_score_metric

392
        output_path = os.path.join(args.results_dir, args.dataset_name, 'stage4')
Charly Lamothe's avatar
Charly Lamothe committed
393
394
395
396
        pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)

        Plotter.plot_stage2_losses(
            file_path=output_path + os.sep + 'losses.png',
Léo Bouscarrat's avatar
Léo Bouscarrat committed
397
398
399
            all_experiment_scores=[base_with_params_test_scores, random_with_params_test_scores, omp_with_params_test_scores,
                                   omp_with_params_without_weights_test_scores],
            all_labels=['base', 'random', 'omp', 'omp_without_weights'],
Charly Lamothe's avatar
Charly Lamothe committed
400
401
402
403
            x_value=with_params_extracted_forest_sizes,
            xlabel='Number of trees extracted',
            ylabel=experiments_score_metric,
            title='Loss values of {}\nusing best params of previous stages'.format(args.dataset_name))
404
405
    elif args.stage == 5:
        # Retreive the extracted forest sizes number used in order to have a base forest axis as long as necessary
406
407
408
        extracted_forest_sizes_number = retreive_extracted_forest_sizes_number(args.models_dir, int(args.experiment_ids[1]))
        all_labels = list()
        all_scores = list()
409
410
411
412
413

        # base_with_params
        logger.info('Loading base_with_params experiment scores...')
        base_with_params_train_scores, base_with_params_dev_scores, base_with_params_test_scores, \
            base_with_params_experiment_score_metric = \
414
            extract_scores_across_seeds_and_forest_size(args.models_dir, args.results_dir, int(args.experiment_ids[0]),
415
416
417
418
419
            extracted_forest_sizes_number)
        # random_with_params
        logger.info('Loading random_with_params experiment scores...')
        random_with_params_train_scores, random_with_params_dev_scores, random_with_params_test_scores, \
            with_params_extracted_forest_sizes, random_with_params_experiment_score_metric = \
420
            extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, int(args.experiment_ids[1]))
421
422
423
424
        # omp_with_params
        logger.info('Loading omp_with_params experiment scores...')
        omp_with_params_train_scores, omp_with_params_dev_scores, omp_with_params_test_scores, _, \
            omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
425
                args.models_dir, args.results_dir, int(args.experiment_ids[2]))
426
427
428
429
        #omp_with_params_without_weights
        logger.info('Loading omp_with_params experiment scores...')
        omp_with_params_without_weights_train_scores, omp_with_params_without_weights_dev_scores, omp_with_params_without_weights_test_scores, _, \
            omp_with_params_experiment_score_metric = extract_scores_across_seeds_and_extracted_forest_sizes(
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
                args.models_dir, args.results_dir, int(args.experiment_ids[2]), weights=False)

        all_labels = ['base', 'random', 'omp', 'omp_without_weights']
        all_scores = [base_with_params_test_scores, random_with_params_test_scores, omp_with_params_test_scores,
            omp_with_params_without_weights_test_scores]

        for i in range(3, len(args.experiment_ids)):
            if 'kmeans' in args.experiment_ids[i]:
                label = 'kmeans'
            elif 'similarity' in args.experiment_ids[i]:
                label = 'similarity'
            elif 'ensemble' in args.experiment_ids[i]:
                label = 'ensemble'
            else:
                logger.error('Invalid value encountered')
                continue
446

447
448
449
450
451
            logger.info(f'Loading {label} experiment scores...')
            _, _, current_test_scores, _, _ = extract_scores_across_seeds_and_extracted_forest_sizes(
                args.models_dir, args.results_dir, int(args.experiment_ids[i].split('=')[1]))
            all_labels.append(label)
            all_scores.append(current_test_scores)
452

453
        output_path = os.path.join(args.results_dir, args.dataset_name, 'stage5')
454
455
456
        pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)

        Plotter.plot_stage2_losses(
457
458
459
            file_path=output_path + os.sep + f"losses_{'-'.join(all_labels)}.png",
            all_experiment_scores=all_scores,
            all_labels=all_labels,
460
461
            x_value=with_params_extracted_forest_sizes,
            xlabel='Number of trees extracted',
462
            ylabel=base_with_params_experiment_score_metric,
463
            title='Loss values of {}\nusing best params of previous stages'.format(args.dataset_name))
464
465
    else:
        raise ValueError('This stage number is not supported yet, but it will be!')
466

467
    logger.info('Done.')