Corrected bug in readtrace

outputs/evalAndStats.py

+#! /usr/bin/env python3
+
+import os
+
+data = "../data/"
+evalScript = "../conll18_ud_eval.py"
+readTrace = "../readTrace.py"
+pvalues = "pvalues/"
+gold = "%sgold/"%pvalues
+
+os.makedirs(gold, exist_ok=True)
+
+groups = {}
+splits = set()
+
+for filename in os.listdir() :
+  if len(filename) < 7 or filename[-7:] != ".conllu" :
+    continue
+  splitNum = int(filename.split("_")[0])
+  splits.add(splitNum)
+  basename = "_".join((".".join(filename.split(".")[:-1])).split("_")[1:])
+  group = basename.split("_")[0]
+  if group not in groups :
+    groups[group] = []
+  groups[group].append(basename)
+
+refs = " ".join(["%sUD_French-GSD_%d/test.conllu"%(data, split) for split in splits])
+os.system("cat %s > %s%s_corpus.conllu"%(refs, gold, ",".join(list(map(str,splits)))))
+  
+for group in groups :
+  groupdir = "%s%s/"%(pvalues, group)
+  os.makedirs(groupdir, exist_ok=True)
+  for elem in groups[group] :
+    ins = " ".join(["%d_%s.conllu"%(split, elem) for split in splits])
+    os.makedirs("%s%s/"%(groupdir, elem), exist_ok=True)
+    os.system("cat %s > %s%s/%s_corpus.conllu"%(ins, groupdir, elem, ",".join(list(map(str,splits)))))
+
+  traces = " ".join(["%d_%s.trace"%(list(splits)[0], elem) for elem in groups[group]])
+  os.system("%s %s --stats > stats_%s.txt"%(readTrace, traces, group))
+    

outputs/pvalues/pvalues.sh

+#!/bin/bash
+NAME='tagparser'
+SYSTEMS=`ls $NAME`
+METRICS="UAS UPOS"
+RUNS=10000
+
+for metric in $METRICS; do
+  [[ $# -ne 0 ]] && SYSTEMS=$@
+  set -x
+  set -e
+  for sys in $SYSTEMS; do
+      mkdir -p results/$sys
+      for testset in `ls $NAME/$sys`; do
+          udapy read.Conllu zone=gold files=gold/$testset \
+                read.Conllu zone=pred files=$NAME/$sys/$testset ignore_sent_id=1 \
+                util.ResegmentGold \
+                eval.Conll18 print_results=0 print_raw=$metric \
+                > results/$sys/${testset%.conllu}
+      done
+  done
+  echo $metric
+  python3 `python3 -c 'import conll18 as x; print(x.__file__)'` -r $RUNS
+  echo ""
+done

readTrace.py

@@ -284,7 +284,7 @@ class History() :
       globalStats["arcsAccuracy"] = 100.0*(globalStats["nbArcs"]-globalStats["nbMissedArcs"])/globalStats["nbArcs"]
     if globalStats["nbErr"] > 0 :
       globalStats["avgErrCost"] /= globalStats["nbErr"]
-    if globalStats["nbErr"] > 0 :
+    if globalStats["nbErrParser"] > 0 :
       globalStats["avgErrCostParser"] /= globalStats["nbErrParser"]
     if globalStats["nbErr"] > 0 :
       globalStats["backRecall"] = 100.0*globalStats["nbErrFound"] / globalStats["nbErr"]