Skip to content
Snippets Groups Projects
Commit f4bf37cf authored by Franck Dary's avatar Franck Dary
Browse files

Passe par conllu et supprimé phrases avec plusieurs racines

parent 51956833
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
fr_orpheo/data/Makefile
+ 9
− 43
View file @ f4bf37cf
TOOLS=../../tools
ORFEO_DIR=../../data/fr_orpheo
MCD=wpmlgfs.mcd
CONLLUMCD=conllu.mcd
MCFMCD=mcf.mcd
#This part is for lemmatizer rules and excpetions computation
THRESHOLD=10
STRICT=-s
FPLM_FILENAME=fplm
RULES_FILENAME=maca_trans_lemmatizer_rules.txt
EXCEPTIONS_FPLM_FILENAME=maca_trans_lemmatizer_exceptions.fplm
FPLM_FILENAME=fplm
FP_FILENAME=fP
compile: fplm fP $(RULES_FILENAME)
train.mcf :
conllu :
./prepareOrfeoData.py $(ORFEO_DIR)/mcf/ $(ORFEO_DIR)/meta_data/
$(TOOLS)/conllu2mcf.py train.conll $(CONLLUMCD) train.mcf $(MCFMCD)
$(TOOLS)/conllu2mcf.py test.conll $(CONLLUMCD) test.mcf $(MCFMCD)
cat train.mcf | ./add_time_and_speaker.perl > tmpTrain
cat test.mcf | ./add_time_and_speaker.perl > tmpTest
mv tmpTest test.mcf
$(TOOLS)/mcfShuffleAndMakeDev.py tmpTrain $(MCD) 0.1 train.mcf dev.mcf
rm tmpTrain
$(RULES_FILENAME): $(FPLM_FILENAME)
macaon_compute_l_rules -f $(FPLM_FILENAME) -e $(EXCEPTIONS_FPLM_FILENAME) -r $(RULES_FILENAME) $(STRICT) -t $(THRESHOLD)
$(FPLM_FILENAME): train.mcf
cat train.mcf dev.mcf test.mcf > all.mcf
$(TOOLS)/mcf2fplm.py all.mcf $(MCD) > tmp
./purge_sfplm.pl $(ORFEO_DIR)/sfplm/ | sort | uniq > tmp2
cat tmp tmp2 fplm_add | sort | uniq | sed '/^$$/d' > $@
rm all.mcf
rm tmp
rm tmp2
$(FP_FILENAME): $(FPLM_FILENAME)
$(TOOLS)/fplm2fP.py $< > $@
./rmBlankLines.py train.conllu test.conllu
~/macaon_data/scripts/conlluCheckProblems.py train.conllu > train 2> pbTrain
~/macaon_data/scripts/conlluCheckProblems.py test.conllu > test 2> pbTest
mv train train.conllu
mv test test.conllu
~/oculometry/scripts/splitTrainDevTest.py --dev 0.1 --test 0.0 train.conllu
clean:
- rm $(RULES_FILENAME)
- rm $(EXCEPTIONS_FPLM_FILENAME)
- rm fP
- rm fplm
- rm *\.mcf
- rm *\.conll*
- rm pbTrain
- rm pbTest
fr_orpheo/data/add_time_and_speaker.perl deleted 100755 → 0
+ 0
− 61
View file @ 51956833
#! /usr/bin/perl
#bonjour INT bonjour -1 dm 0 5.45 5.59 appelant
$first = 1;
while(<>){
chop;
($filename, $word, $pos, $lemma, $head, $label, $start, $end, $speaker, $nbLocuteurs, $milieu, $type, $secteur, $eos) = split /\t/;
if($first){
$first = 0;
$intervalle_int = 0;
$change_speaker = 0;
}
else{
# print "end prec = $end_prec start =
$intervalle = $start - $end_prec;
# if($intervalle < 0){ $intervalle_int = 0;}
# elsif($intervalle < 0.005){ $intervalle_int = 0;}
# elsif($intervalle < 0.065){ $intervalle_int = 1;}
# elsif($intervalle < 0.255){ $intervalle_int = 2;}
# elsif($intervalle < 1.625){ $intervalle_int = 3;}
# else {$intevalle_int = 4;}
# print("intervalle = $intervalle intervalle_int = $intervalle_int\n");
$intervalle_int = int($intervalle * 10);
##alexis 23/01/18
# if($intervalle_int < 0){
# $intervalle_int = 0;
# }
##end
if($intervalle_int < -10){
$intervalle_int = -10;
}
if($intervalle_int > 10){
$intervalle_int = 10;
}
if($speaker ne $speaker_prec){
$change_speaker = 1;
}
else{
$change_speaker = 0;
}
# print "$word_prec\t$intervalle_int\t$change_speaker\t$pos_prec\t$lemma_prec\t$head_prec\t$label_prec\t$eos_prec\n";
}
# print "$word\t$pos\t$lemma\t$head\t$label\t$eos\t$intervalle_int\t$change_speaker\n";
print "$filename\t$word\t$intervalle_int\t$change_speaker\t$pos\t$lemma\t$head\t$label\t$nbLocuteurs\t$milieu\t$type\t$secteur\t$eos\n";
$word_prec = $word;
$pos_prec = $pos;
$lemma_prec = $lemma;
$head_prec = $head;
$label_prec = $label;
$eos_prec = $eos;
$start_prec = $start;
$end_prec = $end;
$speaker_prec = $speaker;
}
fr_orpheo/data/fplm_add deleted 100644 → 0
+ 0
− 26
View file @ 51956833
des prep de #####
des det un #####
de det un #####
aux prep à #####
au prep à #####
du prep de #####
M titre M #####
m titre M #####
Mr titre M #####
mr titre M #####
MM titre M #####
mm titre M #####
Mme titre M #####
mme titre M #####
Mmes titre M #####
mmes titre M #####
Mlle titre M #####
mlle titre M #####
Mlles titre M #####
mlles titre M #####
Dr titre docteur #####
Drs titre docteur #####
Pr titre professeur #####
Prs titre professeur #####
Mgr titre monseigneur #####
mgr titre monseigneur #####
fr_orpheo/data/mcf.mcd deleted 100644 → 0
+ 0
− 14
View file @ 51956833
0 FILENAME
1 FORM
2 POS
3 LEMMA
4 GOV
5 LABEL
6 TIME1
7 TIME2
8 SPKR
9 NBLOCUTEURS
10 MILIEU
11 TYPE
12 SECTEUR
13 EOS
fr_orpheo/data/prepareOrfeoData.py
+ 9
− 2
View file @ f4bf37cf
......@@ -3,6 +3,9 @@
import sys
import os
def mcd() :
return "# global.columns = FILE ID FORM LEMMA POS UPOS FEATS HEAD DEPREL NONE1 NONE2 TIME1 TIME2 SPEAKER NBLOCS TYPE MILIEU"
def printUsageAndExit() :
print("USAGE : %s rawMcfDirectory metaDataDirectory"%(sys.argv[0]), file=sys.stderr)
exit(1)
......@@ -55,8 +58,9 @@ def treatDirectory(mcfs, metadatas) :
metas[name] = {}
metas[name][corresp] = target
output = open("train.conll", "w")
output = open("train.conllu", "w")
print(mcd(), file=output)
for mcf in trains :
featsForFile = list.copy(features)
name = mcf.split(".")[0]
......@@ -77,14 +81,16 @@ def treatDirectory(mcfs, metadatas) :
for line in open(mcfs+mcf, "r") :
clean = line.strip()
if len(line) <= 2 :
print(file=output)
continue
completeLine = clean
for feat in featsForFile :
completeLine += "\t" + feat
print(completeLine,file=output)
output = open("test.conll", "w")
output = open("test.conllu", "w")
print(mcd(), file=output)
for mcf in tests :
featsForFile = list.copy(features)
name = mcf.split(".")[0]
......@@ -105,6 +111,7 @@ def treatDirectory(mcfs, metadatas) :
for line in open(mcfs+mcf, "r") :
clean = line.strip()
if len(line) <= 2 :
print(file=output)
continue
completeLine = clean
for feat in featsForFile :
......
fr_orpheo/data/purge_sfplm.pl deleted 100755 → 0
+ 0
− 62
View file @ 51956833
#!/usr/bin/perl
my $dir= shift;
my $statut;
my $reste;
my @fichiers =(
"$dir/ADJ.sfplm",
"$dir/ADV.sfplm",
"$dir/CLN.sfplm",
"$dir/CSU.sfplm",
"$dir/INT.sfplm",
"$dir/PCT.sfplm",
"$dir/PRO.sfplm",
"$dir/VNF.sfplm",
"$dir/VPR.sfplm",
"$dir/ADN.sfplm",
"$dir/CLI.sfplm",
"$dir/CLS.sfplm",
"$dir/COO.sfplm",
"$dir/DET.sfplm",
"$dir/NOM.sfplm",
"$dir/PRE.sfplm",
"$dir/PRQ.sfplm",
"$dir/VPP.sfplm",
"$dir/VRB.sfplm"
);
foreach $fichier_sfplm (@fichiers){
$fichier_fplm = $fichier_sfplm;
# $fichier_fplm =~ s/sfplm/fplm/;
print STDERR "processing $fichier_sfplm\n";
# print "fichier sfplm = $fichier_sfplm\n";
# print "fichier fplm = $fichier_fplm\n";
# open(FPLM,">$fichier_fplm");
open(SFPLM,"<$fichier_sfplm");
while(<SFPLM>){
s/ //g;
s/ +$//;
s/\t+$//;
s/\t +/\t/g;
s/ +\t/\t/g;
s/\t+/\t/g;
/([^\t]*)\t(.*)/;
$statut = $1;
$reste = $2;
if(($statut eq "N") ||($statut eq "A")){
# print FPLM "$reste\n";
# if((!$reste=~ /\tde\t/)
# && (!$reste=~ /\tdu\t/)
# && (!$reste=~ /\td\'\t/)
# && (!$reste=~ /\tdes\t/)){
print "$reste\n";
# }
}
}
# close FPLM;
close SFPLM;
}
fr_orpheo/data/rmBlankLines.py 0 → 100755
+ 16
− 0
View file @ f4bf37cf
#! /usr/bin/env python3
import sys
for filename in sys.argv[1:] :
lines = []
for line in open(filename, "r") :
line = line.strip()
if len(line) == 0 and len(lines) > 0 and len(lines[-1]) == 0 :
continue
lines.append(line)
with open(filename, "w") as out :
for line in lines :
print(line, file=out)
fr_orpheo/data/wpmlgfs.mcd deleted 100644 → 0
+ 0
− 13
View file @ 51956833
0 FILENAME
1 FORM
2 SPKRCHANGE
3 SILENCE
4 POS
5 LEMMA
6 GOV
7 LABEL
8 NBLOCUTEURS
9 MILIEU
10 TYPE
11 SECTEUR
12 EOS
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment