#!/usr/bin/env python3 import sys import argparse import collections import pdb from conllulib import CoNLLUReader, Util ################################################################################ parser = argparse.ArgumentParser(description="Calculates the accuracy of a \ prediction with respect to the gold file. By default, uses UPOS, but this can \ be configured with option -c.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-D', "--debug", action="store_true", dest="DEBUG_FLAG", help="""Print debug information (grep it or pipe into `less -SR`)""") parser.add_argument('-p', "--pred", metavar="FILENAME.conllu", required=True,\ dest="pred_filename", type=argparse.FileType('r', encoding='UTF-8'), \ help="""Test corpus in CoNLLU with *predicted* tags. (Required)""") parser.add_argument('-g', "--gold", metavar="FILENAME.conllu", required=True,\ dest="gold_filename", type=argparse.FileType('r', encoding='UTF-8'), \ help="""Test corpus in CoNLLU with *gold* tags. (Required)""") parser.add_argument('-t', "--train", metavar="FILENAME.conllu", required=False,\ dest="train_filename", type=argparse.FileType('r', encoding='UTF-8'), \ help="""Training corpus in CoNLL-U, from which tagger was learnt.""") parser.add_argument('-c', "--tagcolumn", metavar="NAME", dest="col_name_tag", required=False, type=str, default="upos", help="""Column name of tags, \ as defined in header. Use lowercase""") parser.add_argument('-f', "--featcolumn", metavar="NAME", dest="col_name_feat", required=False, type=str, default="form", help="""Column name of input feature, as defined in header. Use lowercase.""") parser.add_argument('-u', "--upos-filter", metavar="NAME", dest="upos_filter", required=False, type=str, nargs='+', default=[], help="""Only calculate accuracy for words with UPOS in this list. \ Empty list = no filter.""") ######################################################################## def process_args(parser): args = parser.parse_args() Util.DEBUG_FLAG = args.DEBUG_FLAG args.col_name_tag = args.col_name_tag.lower() args.col_name_feat = args.col_name_feat.lower() Util.debug("Command-line arguments and defaults:") for (k,v) in vars(args).items(): Util.debug(" * {}: {}",k,v) gold_corpus = CoNLLUReader(args.gold_filename) pred_corpus = CoNLLUReader(args.pred_filename) train_vocab = None if args.train_filename: train_corpus = CoNLLUReader(args.train_filename) ignoreme, train_vocab = train_corpus.to_int_and_vocab({args.col_name_feat:[]}) if args.col_name_tag not in gold_corpus.header or \ args.col_name_feat not in gold_corpus.header: Util.error("-c and -f names must be valid conllu column among:\n{}", gold_corpus.header) return args, gold_corpus, pred_corpus, train_vocab ######################################################################## if __name__ == "__main__": args, gold_corpus, pred_corpus, train_vocab = process_args(parser) total_tokens = correct_tokens = 0 total_oov = correct_oov = 0 for (sent_gold, sent_pred) in zip(gold_corpus.readConllu(), pred_corpus.readConllu()): for (tok_gold, tok_pred) in zip (sent_gold, sent_pred): if not args.upos_filter or tok_gold['upos'] in args.upos_filter : if train_vocab : train_vocab_feat = train_vocab[args.col_name_feat].keys() if tok_gold[args.col_name_feat] not in train_vocab_feat: total_oov = total_oov + 1 oov = True else: oov = False if tok_gold[args.col_name_tag] == tok_pred[args.col_name_tag]: correct_tokens = correct_tokens + 1 if train_vocab and oov : correct_oov = correct_oov + 1 total_tokens += 1 print("Pred file: {}".format(pred_corpus.name())) if args.upos_filter : print("Results focus only on following UPOS: {}".format(" ".join(args.upos_filter))) accuracy = (correct_tokens / total_tokens) * 100 print("Accuracy on all {}: {:0.2f} ({}/{})".format(args.col_name_tag,accuracy, correct_tokens, total_tokens)) if train_vocab : accuracy_oov = (correct_oov / total_oov) * 100 print("Accuracy on OOV {}: {:0.2f} ({}/{})".format(args.col_name_tag, accuracy_oov, correct_oov, total_oov))