Commit 7a8bdeea authored by valentin.emiya's avatar valentin.emiya
Browse files

Merge branch 'tffm' of gitlab.lis-lab.fr:skmad-suite/tff2020 into py

parents 408fca50 4cda68d8
Pipeline #6075 passed with stage
in 5 minutes and 42 seconds
......@@ -8,11 +8,13 @@ The sound material is available in folder 'data'.
The code is available in folders 'matlab' and 'python'. The main experiments are available in both programming languages:
* Figure 1 can be reproduced in Matlab by running file `run_illustration_cuicui_eigenvalues.m`
* Figure 2 can be reproduced in Matlab by running file `exp_eigenval_win.m`.
* Figure 3 can be reproduced in Matlab by running file `rank_estimation_halko_vs_eigs_gausswin.m`.
* Figure 1 and 2 can be reproduced in Matlab by running `tff2020/matlab/tfgm/scripts/exp_gabmul_eigs_properties.m`.
* Figure 3 can be reproduced in Matlab by running file `tff2020/matlab/tfgm/scripts/rank_estimation_halko_vs_eigs_gausswin.m`.
* Figure 4 can be reproduced in Python by running the specific tasks 12 and 13 from `tffpy.scripts.script_exp_solve_tff.py`.
* Figure 5 can be reproduced in Python by running the specific tasks 12 and 13 from `tffpy.scripts.script_exp_solve_tff.py`.
* Figure 6 can be reproduced in Python by running the full experiment from `tffpy.scripts.script_exp_solve_tff.py`.
* Table I can be reproduced in Python by running the full experiment from `tffpy.scripts.script_exp_solve_tff.py`.
* Table II can be reproduced in Python by running the full experiment from `tffpy.scripts.script_exp_solve_tff.py`.
......@@ -7,13 +7,13 @@ filtering out. More precisely, it is a Matlab implementation of the algorithms
from *Time-frequency fading algorithms based on Gabor multipliers, by,
A. Marina Krémé, Valentin Emiya, Caroline Chaux, and Bruno Torrésani, 2020*.
For more information please contact ama-marina.kreme@univ-amu.fr
For more information please contact ama-marina.kreme@univ-amu.fr or valentin.emiya@lis-lab.fr
## Installation
Download the folder "tff2020" into the directory of your choice.
Then within MATLAB go to file >> Set path... and add the directory containing
"tff2020/matlab" to the list (if it isn't already).
"tff2020/matlab" to the list (if it isn't already).
## Dependencies
......@@ -25,23 +25,34 @@ which can be downloaded at https://ltfat.github.io
See the documentation.
To reproduce figures on the aforementioned paper, go to the
following directory : "tfgm/scripts" and then run the file
**solve_1area_cuicui.m**
To reproduce the aforementioned paper figures:
You can also run the scripts **solve_all_tff1.m**,
**solve_all_tffP.m** for more experiments
- Figure 1 and 2 can be reproduced by running
**tff2020/matlab/tfgm/scripts/exp_gabmul_eigs_properties.m**
- Figure 3 can be reproduced by running
**tff2020/matlab/tfgm/scripts/rank_estimation_halko_vs_eigs_gausswin.m**
- Figure 4 and 5 can be reproduced by running
**tff2020/matlab/tfgm/scripts/exp_tff1_car_bird.m**
You can also run the scripts
**tff2020/matlab/tfgm/scripts/exp_all_tff1.m** and
**tff2020/matlab/tfgm/scripts/exp_all_tffP.m**
for more experiments
## Copyright © 2019-2020
- [Laboratoire d'Informatique et Systèmes](https://www.lis-lab.fr)
- [Institut de Mathématiques de Marseille](https://www.i2m.univ-amu.fr)
- [Université d'Aix-Marseille](https://www.univ-amu.fr)
- [Laboratoire d'Informatique et Systemes](https://www.lis-lab.fr)
- [Institut de Mathematiques de Marseille](https://www.i2m.univ-amu.fr)
- [Universite d'Aix-Marseille](https://www.univ-amu.fr)
## Contributors
- [A. Marina Krémé](ama-marina.kreme@univ-amu.fr)
- [A. Marina Kreme](ama-marina.kreme@univ-amu.fr)
- [Valentin Emiya](valentin.emiya@lis-lab.fr)
......
File added
clc; clear; close all;
%%
% Generate all combination between wide-band and localized signals
%%
make_wav_pairs()
% This script allows to generate all the possible mixtures as well as
% the parameters for the corresponding masks for each window.
%%
pwd;
pathname ='spectro_all_mixtures';
if ~exist('spectro_all_mixtures','dir')
mkdir('spectro_all_mixtures');
pathname ='fig_spectro_all_mixtures';
if ~exist(pathname,'dir')
mkdir(pathname);
end
addpath('spectro_all_mixtures')
addpath(pathname)
%%
data = load('signal_lists.mat');
dataset = get_dataset();
dbstack;
%%
resampling_fs = 8000;
sig_len =16384;
signal_params = generate_signal_parameters(resampling_fs, sig_len);
%% DGT parameters
approx_win_len = 512;
win_len = approx_win_len;
win_type='hann';
params = get_params(win_len, win_type);
dgt_params = generate_dgt_parameters(win_type, approx_win_len, params.hop, params.nbins, sig_len);
%% DGT operators
[dgt, idgt] = get_stft_operators(dgt_params, signal_params);
wins_params = struct('Gauss256', struct('win_type','gauss','win_len', 256),...,
'Hann512', struct('win_type','hann','win_len', 512));
gamma=0.7;
fs = 8000;
sig_len =16384;
signal_params = generate_signal_parameters(fs, sig_len);
%% All mix
close all
target_name ={'car','plane','train'};
per_name = {'beeps','bird','clicks','pop'};
%%
wideband_name ={'car','plane','train'};
localized_name = {'beeps','bird','chirps','clicks','finger_snaps','modulations'};
gamma=0.7;
deb=0;
close all;
for k=1:length(data.wide_band_sources_files)
sig_wd = load_wideband_signal(k, resampling_fs, sig_len);
%% DGT parameters
keys = fieldnames(wins_params);
for win_param = 1: length(keys)
for l=1:length(data.localized_sources_files)
%%
%figure;
sig_loc = load_localized_signal(l, resampling_fs, sig_len,deb);
signals = generate_mix_signal(sig_wd, sig_loc, gamma);
tf_mat_target = compute_dgt(signals.wideband, dgt );
tf_mat_per = compute_dgt(signals.localized, dgt );
tf_mat_mix = compute_dgt(signals.mix, dgt );
[alpha, seuil, radius] = set_smooth_mask_params(target_name{k}, per_name{l}, win_type);
sdr_mix = sdr(signals.wideband, signals.mix);
[original_mask, mask_after_imclose, mask_after_imopen,...,
mask] = generate_mask(tf_mat_target, tf_mat_per, alpha, seuil, radius);
subplot(221);
plot_spectrogram(tf_mat_target, dgt_params, signal_params, dgt )
title(target_name{k})
subplot(222);
plot_spectrogram(tf_mat_per, dgt_params, signal_params, dgt )
title(per_name(l));
subplot(223);
plot_spectrogram(tf_mat_mix, dgt_params, signal_params, dgt )
title(['Mix SDR= ',num2str(sdr_mix),'dB']);
subplot(224);
plot_spectrogram(mask, dgt_params, signal_params, dgt )
title(['mask - ', num2str(win_type)]);
saveas(gcf,fullfile(pathname, [target_name{k},'_' num2str(l), '_',num2str(win_type) '.png']));
% figure;
% plot_spectrogram((1-mask).*tf_mat_per, dgt_params, signal_params, dgt );
% title(['masked spectro - ', target_name{k}, per_name(l)])
%saveas(gcf,fullfile(pathname, ['masked_spectro_', target_name{k},'_' num2str(l), '_',num2str(win_type) '.png']));
end
win_len = wins_params.(keys{win_param}).win_len;
win_type = wins_params.(keys{win_param}).win_type;
params = get_params(win_len, win_type);
fprintf("window %s:\n\n",win_type);
% DGT parameters
dgt_params = generate_dgt_parameters(win_type, win_len, params.hop,...,
params.nbins, sig_len);
%DGT operators
[dgt, idgt] = get_stft_operators(dgt_params, signal_params);
for wb = 1:length(wideband_name)
wideband_src =wideband_name{wb};
for loc = 1:length(localized_name)
loc_source = localized_name{loc};
[x_loc, fs_loc]= audioread(dataset.localized.(loc_source));
[x_wb, fs_wb]= audioread(dataset.wideband.(wideband_src));
if length(x_loc)~=length(x_wb)
warning('Arrays are not equal');
end
if fs_loc~=fs_wb
error('The sampling frequencies must be the same.')
end
signals = generate_mix_signal(x_wb, x_loc, gamma);
tf_mat_wb = dgt(signals.wideband);
tf_mat_loc = dgt(signals.localized);
tf_mat_mix = dgt(signals.mix);
[alpha, thres, radius] = set_smooth_mask_params(wideband_src,...,
loc_source, win_type);
[mask] = generate_mask(tf_mat_wb, tf_mat_loc, alpha, thres, radius);
figure ;
subplot(221)
plot_spectrogram(tf_mat_wb, dgt_params, signal_params, dgt );
title('wideband')
subplot(222)
plot_spectrogram(tf_mat_loc, dgt_params, signal_params, dgt );
title('localized')
subplot(223)
plot_spectrogram(tf_mat_mix, dgt_params, signal_params, dgt );
title('mix')
subplot(224)
plot_spectrogram(mask, dgt_params, signal_params, dgt );
title('mask')
end
end
end
function [signal_params, dgt_params, w,dgt,idgt, signals, alpha, seuil,...,
r, mask,dgt_mix]= get_cuicui_mix_icassp()
%% This function allows you to reproduce ICASSP dataset.
%% Signal parameters
%make_wav_pairs();
sig_len = 2^(13);
fs = 8000;
signal_params.fs = fs;
signal_params.sig_len = sig_len;
%% dgt parameters
approx_win_duration =0.02;
win_type ='hann';
win_len = 2^(round(log2(approx_win_duration*fs)));
dgt_params.hop = win_len/4 ;
dgt_params.nbins=4*win_len;
dgt_params.sig_len = sig_len;
dgt_params.win_type = win_type;
dgt_params.win_len = win_len;
%% window and DGT operator
L=dgtlength(dgt_params.sig_len, dgt_params.hop, dgt_params.nbins);
w = gabwin(dgt_params.win_type, dgt_params.hop, dgt_params.nbins, dgt_params.win_len);
dgt = @(x) dgtreal(x,w, dgt_params.hop, dgt_params.nbins, dgt_params.sig_len);
wd = {'dual', dgt_params.win_type};
idgt= @(x)idgtreal(x, wd, dgt_params.hop,dgt_params.nbins, dgt_params.sig_len);
compute_stft = @(x,w,dgt_params,L) dgtreal(x,w, dgt_params.hop, dgt_params.nbins,L);
%% signals and their mixtures
ind_wideband = 1;
ind_localized = 2;
deb = 0.2;
[x_engine, x_bird] = load_pairs(ind_wideband, ind_localized,...,
fs, sig_len, deb);
%%
x_target = x_engine;
x_perturbation = x_bird;
gamma=0.7;
x_target = x_target/max(abs(x_target));
x_perturbation = x_perturbation/max(abs(x_perturbation));
x_target = gamma*x_target;
x_perturbation = (1-gamma)*x_perturbation;
x_mix = x_target +x_perturbation;
signals.target = x_target;
signals.noise = x_perturbation;
signals.mix = x_mix;
%% Spectogramm of signals
dgt_Xref = compute_stft(signals.target,w, dgt_params,L);
dgt_Xper = compute_stft(signals.noise, w, dgt_params,L);
dgt_mix = dgt_Xref +dgt_Xper;
%% mask
alpha = 2; seuil =0.02; r = 3;
mask = and(abs(dgt_Xper) < alpha*abs(dgt_Xref),abs(dgt_Xper)<seuil);
se = strel('disk',r);
mask =imclose(mask,se);
figure; imagesc(mask);
%% plot figures
fig_dir ='figures_car_bird_icassp';
if ~exist(fig_dir,'dir')
mkdir(fig_dir);
end
addpath(fig_dir)
figure;
plot_spectrogram(mask, dgt_params, signal_params, dgt);
title('mask')
saveas(gcf,fullfile(fig_dir,'mask.png'));
figure;
plot_spectrogram(dgt_Xref, dgt_params, signal_params, dgt)
title('target signal')
saveas(gcf,fullfile(fig_dir,'engine.png'));
figure;
plot_spectrogram(dgt_Xper, dgt_params, signal_params, dgt)
title('perturbation signal')
saveas(gcf,fullfile(fig_dir,'bird.png'));
figure;
plot_spectrogram(dgt_mix, dgt_params, signal_params, dgt)
title('perturbation signal')
saveas(gcf,fullfile(fig_dir,'engine_bird.png'));
function dataset = get_dataset()
% This function is used to get the data of the experiments.
%The data is stored in a structure array.
% Get dataset for isolated wideband and localized sources before mixing.
% dataset : struct
% dataset.('wideband').(sounds names)is a dictionary
% containing the path of all the wideband and localized sounds.
%
% Author: Marina KREME
......@@ -13,7 +16,7 @@ n_loc_dir = dir([loc_dir, '*.wav']);
dataset = struct('wideband',struct(),'localized',struct());
for x =1: length(n_wide_dir)
for x =1:length(n_wide_dir)
wideband_file = [n_wide_dir(x).folder, filesep,n_wide_dir(x).name];
wideband_name = n_wide_dir(x).name(1:end-4);
......@@ -35,4 +38,3 @@ end
......@@ -3,27 +3,29 @@ function [signals, dgt_params, signal_params, mask, mask_area, dgt,...,
nbins_ratio, win_type, alpha, thres, radius, fig_dir)
%%
% Function that generates :
% - the mixture of a wide-band spectrogram signal and a localized signal.
% - the mask from the two mixtures
% Build the mix two sounds and the related time-frequency boolean mask.
%
%
% Inputs:
% - loc_source(str): signal with localized spectrogram
% - wideband_src(str): signal with wideband spectrogram
% - gamma: integer (belong to ]0,1[)
% - win_dur: window duration (must be between 12 and 20 ms)
% - hop_ratio, nbins_ration: real
% - hop_ratio (float): Ratio of the window length that will be set as hop size for the DGT.
% - nbins_ratio: Factor that will be applied to the window length to compute the
% number of bins in the DGT.
%
% - win_type (str): analysis window (hann or gauss)
% -alpha, thres, radius(real): smoothing parameter for the mask
% -fig_dir: directory
% - alpha, thres, radius(real): smoothing parameter for the mask
% -fig_dir: directory - folder where figures are stored
%
% Outputs:
% -signals(struct): contains wideband, localized and mixtures signals
% - dgt_params (struct): contains dgt parameters (hop, n_bins, win_type, win_len, win)
% - signal_params (struct): contains the length of signal and sampling frequency
% - mask : binary mask
% - mask_are: binary mask area
% - dgt, idgt: Operator of Gabor transform and its inverse
% - mask : Time-frequency binary mask
% - mask_area: binary mask area
% - dgt, idgt (handle): Operator of DGT transform and its inverse
%
% Author : Marina KREME
......@@ -46,7 +48,6 @@ fs= fs_loc;
sig_len = length(x_loc);
signal_params = generate_signal_parameters(fs, sig_len);
%% build mix signals
signals = generate_mix_signal(x_wb, x_loc, gamma);
......@@ -56,14 +57,15 @@ approx_win_len = 2.^round(log2(win_dur*fs));
hop = approx_win_len* hop_ratio;
nbins = approx_win_len * nbins_ratio;
dgt_params = generate_dgt_parameters(win_type, approx_win_len, hop, nbins, sig_len);
dgt_params = generate_dgt_parameters(win_type, approx_win_len, hop,...,
nbins, sig_len);
%% generat mask
%% generate binary mask
[dgt, idgt] = get_stft_operators(dgt_params, signal_params);
tf_mat_wb = compute_dgt(signals.wideband, dgt );
tf_mat_loc = compute_dgt(signals.localized, dgt );
tf_mat_wb = dgt(signals.wideband);
tf_mat_loc = dgt(signals.localized);
[mask, original_mask, mask_after_imclose, mask_after_imopen,...,
] = generate_mask(tf_mat_wb, tf_mat_loc, alpha, thres, radius);
......@@ -76,43 +78,55 @@ title(['Mask Mix : mask-area = ',num2str(mask_area)]);
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'mask_mix.pdf'));
%%
figure;
plot_spectrogram(signals.wideband, dgt_params, signal_params, dgt)
title(['Wideband source - ', wideband_src]);
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'mix_spectro_mask.pdf'));
% plot_spectrogram(signals.wideband, dgt_params, signal_params, dgt)
% title(['Wideband source - ', wideband_src]);
subplot(231)
figure;
plot_spectrogram(signals.localized, dgt_params, signal_params, dgt)
title(['Localized source - ',loc_source]);
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'loc_source.pdf'));
subplot(232)
figure;
plot_spectrogram(signals.mix, dgt_params, signal_params, dgt)
title('Mix');
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'mix_spectrogram.pdf'));
subplot(233)
figure;
plot_mask(original_mask, hop, nbins, fs);
title('Original mask');
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'raw_mask.pdf'));
subplot(234)
figure;
plot_mask(mask_after_imclose, hop, nbins, fs);
title('Smooth mask- after imclose');
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'mask_after_imclose.pdf'));
subplot(235)
figure;
plot_mask(mask_after_imopen, hop, nbins, fs);
title('Smooth and final mask - after impoen');
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'mask_after_imopen.pdf'));
m = ~mask;
figure;
m = ~mask;
gabmul = gen_gabmul_operator(dgt, idgt,m);
x_est = gabmul(signals.wideband);
subplot(236)
plot_spectrogram(x_est, dgt_params, signal_params, dgt)
title('Filtered wb')
saveas(gcf,fullfile(fig_dir, 'mix_spectro_mask.pdf'));
set(gca, 'FontSize', 20, 'fontName','Times');
saveas(gcf,fullfile(fig_dir, 'zerofill_spectrogram.pdf'));
end
\ No newline at end of file
function [win_type, win_dur,win_len, hop_ratio, nbins_ratio] = get_win_params(win_choice)
win_choice_split = split(win_choice);
win_type = win_choice_split{1};
win_len_str = win_choice_split{2};
win_len = str2double(win_len_str);
win_dur = win_len/ 8000;
switch win_type
case 'gauss'
hop_ratio = 1 / 4;
nbins_ratio = 4;
case 'hann'
hop_ratio = 1 / 8;
nbins_ratio = 2;
end
end
\ No newline at end of file
clc; clear; close all;
%%
[signal_params, dgt_params, w,dgt, signals, alpha, seuil,...,
r, mask]= get_cuicui_mix_icassp();
\ No newline at end of file
function [alpha, thres, radius] = set_smooth_mask_params(wideband_src, loc_source, win_type)
function [alpha, thres, radius] = set_smooth_mask_params(wideband_src, loc_source, win_type)
% This function allow us to generate parameters for smooth binary mask
% Inputs:
% - wideband_src : signal with wide-bande spectrogram
......@@ -31,6 +31,11 @@ switch win_type
thres=1e-4;
radius=4;
case 'chirps'
alpha=0.1;
thres=0.0001;
radius=1;
case 'clicks'
alpha=1;
thres=0.0002;
......@@ -40,14 +45,12 @@ switch win_type
alpha=0.01;
thres=0.0001;
radius=1;
case 'modulations'
alpha=1;
thres=0.0002;
radius=3;
case 'pop'
alpha=0.1;
thres=0.0001;
radius=1;
end
case'plane'
......@@ -63,22 +66,26 @@ switch win_type
thres=0.0001;
radius=1;
case 'chirps'
alpha=0.01;
thres=0.0001;
radius=1;
case 'clicks'
alpha=1;
thres=0.0002;
radius=4;
case 'finger_snaps'
alpha=0.01;
thres=