diff --git a/eval_all.py b/eval_all.py
index 84f94ae61fe5b770e048ca10b8499e3fbdbe9d77..fadf8d1e103d1a6ffa41b05fe0491b5f4d36607b 100644
--- a/eval_all.py
+++ b/eval_all.py
@@ -12,26 +12,24 @@ cent_thr = 50
metrics = ['recall', 'FA', 'pitch_acc', 'chroma_acc', 'diff_distrib']
parser = argparse.ArgumentParser()
parser.add_argument('specie', type=str, help="Species to treat specifically", default='all')
-parser.add_argument('--drop_noisy_bins', type=bool, help="drop noisy vocalisations", default=False)
-parser.add_argument('--drop_noisy_vocs', type=bool, help="drop noisy STFT bins", default=False)
args = parser.parse_args()
for specie in species if args.specie=='all' else args.specie.split(' '):
- algos = {'pyin', 'praat', 'crepe', 'tcrepe', 'tcrepe_ft', 'tcrepe_ftsp', 'tcrepe_ftoth', 'basic', 'pesto', 'pesto_ft'}
+ algos = {'pyin', 'praat', 'crepe', 'tcrepe', 'tcrepe_ftsp', 'tcrepe_ftoth', 'basic', 'pesto', 'pesto_ft', 'pesto_ftoth'}
# Get optimal thresholds
confs = {k:[] for k in algos}
confs['label'] = []
for fn in tqdm(glob(species[specie]['wavpath'][:-4]+'_preds.csv'), desc=f'{specie} get thrs', leave=False):
- if args.drop_noisy_vocs and os.path.isfile(f'noisy_pngs/{fn[:-10]}.png'):
- continue
df = pd.read_csv(fn)
for algo in algos:
if algo+'_conf' in df.columns:
confs[algo].extend(df[algo+'_conf'])
+ else:
+ confs[algo].extend([np.nan]*len(df))
confs['label'].extend(df.annot>0)
thrs = {}
for algo in list(algos):
- if np.isnan(confs[algo]).all() or len(confs[algo]) < len(confs['label']):
+ if np.isnan(confs[algo]).all():
algos -= {algo}
continue
fpr, tpr, thr = skmetrics.roc_curve(np.array(confs['label'])[~np.isnan(confs[algo])], np.array(confs[algo])[~np.isnan(confs[algo])])
@@ -39,14 +37,8 @@ for specie in species if args.specie=='all' else args.specie.split(' '):
# Compute recall, false alarm, pitch acc and chroma acc
def fun(fn):
- if args.drop_noisy_vocs and os.path.isfile(f'noisy_pngs/{fn[:-10]}.png'):
- return pd.DataFrame()
- df = pd.read_csv(fn).fillna(0)
+ df = pd.read_csv(fn)
df.annot = mir_eval.melody.hz2cents(df.annot)
- if args.drop_noisy_bins and 'salience' in df.columns:
- df.loc[((df.salience < 0.2) | (df.SHR > 10*np.log10(0.2))), 'annot'] = 0
- if not (df.annot > 0).any():
- return pd.DataFrame()
out = pd.DataFrame(columns=metrics)
for algo in algos:
if not algo+'_f0' in df.columns or df[algo+'_f0'].isna().all():
@@ -54,9 +46,10 @@ for specie in species if args.specie=='all' else args.specie.split(' '):
out.loc[algo, ['Recall', 'False alarm']] = mir_eval.melody.voicing_measures(df.annot > 0, df[algo+'_conf'] > thrs[algo])
df[algo+'_f0'] = mir_eval.melody.hz2cents(df[algo+'_f0'])
df[algo+'_conf'].clip(0, 1, inplace=True)
- out.loc[algo, 'Pitch acc'] = mir_eval.melody.raw_pitch_accuracy(df.annot>0, df.annot, df[algo+'_conf'], df[algo+'_f0'], cent_tolerance=50)
- out.loc[algo, 'Chroma acc'] = mir_eval.melody.raw_chroma_accuracy(df.annot>0, df.annot, df[algo+'_conf'], df[algo+'_f0'], cent_tolerance=50)
+ out.loc[algo, 'Pitch acc'] = mir_eval.melody.raw_pitch_accuracy(df.annot>0, df.annot, df[algo+'_conf'], df[algo+'_f0'], cent_tolerance=cent_thr)
+ out.loc[algo, 'Chroma acc'] = mir_eval.melody.raw_chroma_accuracy(df.annot>0, df.annot, df[algo+'_conf'], df[algo+'_f0'], cent_tolerance=cent_thr)
out.at[algo, 'diff_distrib'] = list(abs(df[algo+'_f0'] - df.annot))
+ out.loc[algo, 'Voc. recall'] = ((df.annot > 0 ) & ( df[algo+'_conf'] > thrs[algo])).sum() > 0.5 * (df.annot > 0).sum()
return out
df = pd.concat(p_umap(fun, glob(species[specie]['wavpath'][:-4]+'_preds.csv'), desc=f'{specie} get perf'))
@@ -67,20 +60,20 @@ for specie in species if args.specie=='all' else args.specie.split(' '):
ax[0].violinplot(np.concatenate(df.loc[algo, 'diff_distrib']), positions=[i])
ax[1].violinplot(df.loc[algo, 'Pitch acc'], positions=[i])
ax[2].violinplot(df.loc[algo, 'Chroma acc'], positions=[i])
-
- ax[0].set_yscale('log')
+
+# ax[0].set_yscale('log')
ax[0].set_title('Distrib of errors in cents')
ax[0].hlines(1200, 0, len(algos), linestyle='dashed', color='k')
ax[1].set_title('Distrib of pitch acc per vocs in % ')
ax[2].set_title('Distrib of chroma acc per vocs in % ')
plt.xticks(np.arange(len(algos)), algos, rotation=45)
plt.tight_layout()
- plt.savefig(f'scores/{specie}_report{"_minusvocs" if args.drop_noisy_vocs else ""}{"_minusbins" if args.drop_noisy_bins else ""}.pdf')
+ plt.savefig(f'scores/{specie}_report.pdf')
plt.close()
- df = df.reset_index(names='algo').groupby('algo').agg({'algo':'count', 'Recall':'mean', 'False alarm':'mean', 'Pitch acc':'mean', 'Chroma acc':'mean'})
+ df['Voc. recall'] = df['Voc. recall'].astype(int)
+ df = df.reset_index(names='algo').groupby('algo').agg({'algo':'count', 'Recall':'mean', 'False alarm':'mean', 'Pitch acc':'mean', 'Chroma acc':'mean', 'Voc. recall':'mean'})
df.loc[thrs.keys(), 'threshold'] = list(thrs.values())
df.rename(columns={'algo':'count'}, inplace=True)
print(df)
- df.to_csv(f'scores/{specie}_scores{"_minusvocs" if args.drop_noisy_vocs else ""}{"_minusbins" if args.drop_noisy_bins else ""}.csv')
-# df.to_latex(f'{specie}_scores.tex', float_format=lambda d: f'{d:.2f}')
\ No newline at end of file
+ df.to_csv(f'scores/{specie}_scores.csv')