Few fixes in compute_results

code/compute_results.py

@@ -211,7 +211,7 @@ def extract_correlations_across_seeds(models_dir, results_dir, experiment_id):
             extracted_forest_size_path = extracted_forest_sizes_root_path + os.sep + extracted_forest_size
             # Load models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}/model_raw_results.pickle file
             model_raw_results = ModelRawResults.load(extracted_forest_size_path)
-            experiment_correlations[seed].append(model_raw_results.correlation)
+            experiment_correlations[seed].append(model_raw_results.train_correlation)
 
     return experiment_correlations
 
@@ -239,10 +239,38 @@ def extract_coherences_across_seeds(models_dir, results_dir, experiment_id):
             extracted_forest_size_path = extracted_forest_sizes_root_path + os.sep + extracted_forest_size
             # Load models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}/model_raw_results.pickle file
             model_raw_results = ModelRawResults.load(extracted_forest_size_path)
-            experiment_coherences[seed].append(model_raw_results.coherence)
+            experiment_coherences[seed].append(model_raw_results.train_coherence)
 
     return experiment_coherences
 
+def extract_strengths_across_seeds(models_dir, results_dir, experiment_id):
+    experiment_id_path = models_dir + os.sep + str(experiment_id) # models/{experiment_id}
+    experiment_seed_root_path = experiment_id_path + os.sep + 'seeds' # models/{experiment_id}/seeds
+    experiment_strengths = dict()
+
+    # For each seed results stored in models/{experiment_id}/seeds
+    seeds = os.listdir(experiment_seed_root_path)
+    seeds.sort(key=int)
+    for seed in seeds:
+        experiment_seed_path = experiment_seed_root_path + os.sep + seed # models/{experiment_id}/seeds/{seed}
+        extracted_forest_sizes_root_path = experiment_seed_path + os.sep + 'extracted_forest_sizes' # models/{experiment_id}/seeds/{seed}/forest_size
+
+        # {{seed}:[]}
+        experiment_strengths[seed] = list()
+
+        # List the forest sizes in models/{experiment_id}/seeds/{seed}/extracted_forest_sizes
+        extracted_forest_sizes = os.listdir(extracted_forest_sizes_root_path)
+        extracted_forest_sizes = [nb_tree for nb_tree in extracted_forest_sizes if not 'no_weights' in nb_tree ]
+        extracted_forest_sizes.sort(key=int)
+        for extracted_forest_size in extracted_forest_sizes:
+            # models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}
+            extracted_forest_size_path = extracted_forest_sizes_root_path + os.sep + extracted_forest_size
+            # Load models/{experiment_id}/seeds/{seed}/extracted_forest_sizes/{extracted_forest_size}/model_raw_results.pickle file
+            model_raw_results = ModelRawResults.load(extracted_forest_size_path)
+            experiment_strengths[seed].append(model_raw_results.test_strength)
+
+    return experiment_strengths
+
 def extract_selected_trees_scores_across_seeds(models_dir, results_dir, experiment_id, weighted=False):
     experiment_id_path = models_dir + os.sep + str(experiment_id) # models/{experiment_id}
     experiment_seed_root_path = experiment_id_path + os.sep + 'seeds' # models/{experiment_id}/seeds
@@ -784,14 +812,14 @@ if __name__ == "__main__":
         experiment_weights = extract_weights_across_seeds(args.models_dir, args.results_dir, experiment_id)
         Plotter.weight_density(experiment_weights, os.path.join(root_output_path, f'weight_density_{experiment_label}.png'))
     if args.plot_preds_coherence:
-        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_new')
+        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_27-03-20')
         pathlib.Path(root_output_path).mkdir(parents=True, exist_ok=True)
         all_labels = ['random', 'omp', 'kmeans', 'similarity_similarities', 'similarity_predictions', 'ensemble']
         _, _, _, with_params_extracted_forest_sizes, _ = \
             extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, 2)
         coherence_values = [extract_coherences_across_seeds(args.models_dir, args.results_dir, i) for i in args.experiment_ids]
         Plotter.plot_stage2_losses(
-            file_path=root_output_path + os.sep + f"coherences_{'-'.join(all_labels)}.png",
+            file_path=root_output_path + os.sep + f"coherences_{'-'.join(all_labels)}_train.png",
             all_experiment_scores=coherence_values,
             all_labels=all_labels,
             x_value=with_params_extracted_forest_sizes,
@@ -801,14 +829,14 @@ if __name__ == "__main__":
         logger.info(f'Computing preds coherence plot...')
 
     if args.plot_preds_correlation:
-        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_new')
+        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_27-03-20')
         pathlib.Path(root_output_path).mkdir(parents=True, exist_ok=True)
         all_labels = ['none', 'random', 'omp', 'kmeans', 'similarity_similarities', 'similarity_predictions', 'ensemble']
         _, _, _, with_params_extracted_forest_sizes, _ = \
             extract_scores_across_seeds_and_extracted_forest_sizes(args.models_dir, args.results_dir, 2)
         correlation_values = [extract_correlations_across_seeds(args.models_dir, args.results_dir, i) for i in args.experiment_ids]
         Plotter.plot_stage2_losses(
-            file_path=root_output_path + os.sep + f"correlations_{'-'.join(all_labels)}.png",
+            file_path=root_output_path + os.sep + f"correlations_{'-'.join(all_labels)}_train.png",
             all_experiment_scores=correlation_values,
             all_labels=all_labels,
             x_value=with_params_extracted_forest_sizes,
@@ -818,7 +846,7 @@ if __name__ == "__main__":
         logger.info(f'Computing preds correlation plot...')
 
     if args.plot_forest_strength:
-        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_strength')
+        root_output_path = os.path.join(args.results_dir, args.dataset_name, f'stage5_27-03-20')
         pathlib.Path(root_output_path).mkdir(parents=True, exist_ok=True)
 
         _, _, _, with_params_extracted_forest_sizes, _ = \
@@ -837,7 +865,7 @@ if __name__ == "__main__":
         #random_selected_trees_scores = extract_selected_trees_scores_across_seeds(
         #    args.models_dir, args.results_dir, 2, weighted=True)
 
-        omp_selected_trees_scores = extract_selected_trees_scores_across_seeds(
+        """omp_selected_trees_scores = extract_selected_trees_scores_across_seeds(
             args.models_dir, args.results_dir, 3, weighted=True)
 
         similarity_similarities_selected_trees_scores = extract_selected_trees_scores_across_seeds(
@@ -847,27 +875,25 @@ if __name__ == "__main__":
         #    args.models_dir, args.results_dir, 7)
 
         ensemble_selected_trees_scores = extract_selected_trees_scores_across_seeds(
-            args.models_dir, args.results_dir, 8, weighted=True)
+            args.models_dir, args.results_dir, 8, weighted=True)"""
 
         # kmeans=5
         # similarity_similarities=6
         # similarity_predictions=7
         # ensemble=8
 
-        all_selected_trees_scores = [random_selected_trees_scores, omp_selected_trees_scores, similarity_similarities_selected_trees_scores,
-            ensemble_selected_trees_scores]
+        all_labels = ['random', 'omp', 'kmeans', 'similarity_similarities', 'similarity_predictions', 'ensemble']
+        strengths_values = [extract_strengths_across_seeds(args.models_dir, args.results_dir, i) for i in args.experiment_ids]
 
-        with open('california_housing_forest_strength_scores.pickle', 'wb') as file:
-            pickle.dump(all_selected_trees_scores, file)
+        """with open('california_housing_forest_strength_scores.pickle', 'wb') as file:
+            pickle.dump(all_selected_trees_scores, file)"""
 
         """with open('forest_strength_scores.pickle', 'rb') as file:
             all_selected_trees_scores = pickle.load(file)"""
 
-        all_labels = ['random', 'omp', 'similarity_similarities', 'ensemble']
-
         Plotter.plot_stage2_losses(
-            file_path=root_output_path + os.sep + f"forest_strength_{'-'.join(all_labels)}_v2_sota.png",
-            all_experiment_scores=all_selected_trees_scores,
+            file_path=root_output_path + os.sep + f"forest_strength_{'-'.join(all_labels)}.png",
+            all_experiment_scores=strengths_values,
             all_labels=all_labels,
             x_value=with_params_extracted_forest_sizes,
             xlabel='Number of trees extracted',