highlight last pronounced line

README

@@ -13,3 +13,12 @@ Run:
 
 python2 main.py
 
+Doc:
+
+developing with pygtk3: http://lazka.github.io/pgi-docs/, https://python-gtk-3-tutorial.readthedocs.org/en/latest/
+
+Todo:
+
+- use GtkSourceView to allow editing the xml file directly
+- model selection in config file
+- integrate new xml with actions

asr.py

@@ -9,7 +9,7 @@ Gdk.threads_init()
 Gst.init(None)
 
 class ASR(Gtk.HBox):
-    def __init__(self, hyp_callback=None):
+    def __init__(self, hyp_callback = None, partial_hyp_callback = None):
         super(ASR, self).__init__()
 
         self.text = Gtk.TextView()
@@ -33,6 +33,7 @@ class ASR(Gtk.HBox):
 
         self.hyp = []
         self.hyp_callback = hyp_callback
+        self.partial_hyp_callback = partial_hyp_callback
         Thread(target=self.init_gst).start()
 
     def init_gst(self):
@@ -53,9 +54,9 @@ class ASR(Gtk.HBox):
           if not os.path.isfile(model_file):
               print >> sys.stderr, "Models not downloaded? Run prepare-models.sh first!"
               sys.exit(1)
-          self.asr.set_property("fst", "asr/HCLG.fst")
+          self.asr.set_property("fst", "asr/model2/HCLG.fst")
           self.asr.set_property("model", "asr/final.mdl")
-          self.asr.set_property("word-syms", "asr/words.txt")
+          self.asr.set_property("word-syms", "asr/model2/words.txt")
           self.asr.set_property("feature-type", "mfcc")
           self.asr.set_property("mfcc-config", "asr/conf/mfcc.conf")
           self.asr.set_property("ivector-extraction-config", "asr/conf/ivector_extractor.fixed.conf")
@@ -120,8 +121,8 @@ class ASR(Gtk.HBox):
         if len(self.hyp) == 0:
             self.hyp = ['']
         self.hyp[-1] = hyp
-        if self.hyp_callback:
-            self.hyp_callback(self.hyp)
+        if self.partial_hyp_callback:
+            self.partial_hyp_callback(self.hyp)
 
         hyp += '...'
         self.insert = self.buffer.get_iter_at_line(self.buffer.get_line_count() - 1)
@@ -135,7 +136,6 @@ class ASR(Gtk.HBox):
         if len(self.hyp) == 0:
             self.hyp = ['']
         self.hyp[-1] = hyp
-        self.hyp.append('')
         if self.hyp_callback:
             self.hyp_callback(self.hyp)
 
@@ -143,6 +143,7 @@ class ASR(Gtk.HBox):
         self.buffer.delete(self.insert, self.buffer.get_end_iter())
         self.buffer.insert(self.insert, hyp + '\n...')
 
+        self.hyp.append('')
         Gdk.threads_leave()
 
     def autoscroll(self, *args):

data/style.css

@@ -6,6 +6,11 @@
 .text-line {
 }
 
+.highlighted {
+    color: red;
+    font: bold;
+}
+
 .section-title {
     font: bold 18;
     color: white;

main.py

@@ -39,6 +39,8 @@ class ScriptedASR(Gtk.Window):
 
         self.xmlview = xmlview_widgets.XmlView(xml_filename)
         vbox.pack_start(self.xmlview, True, True, 5)
+        self.lines = [x for x in self.xmlview.get_line_iterator()]
+        self.current_line = -1
 
         self.confirmer = confirm.ConfirmationBox()
         vbox.pack_start(self.confirmer, False, True, 5)
@@ -64,8 +66,26 @@ class ScriptedASR(Gtk.Window):
         Gtk.StyleContext.add_provider_for_screen( Gdk.Screen.get_default(), style_provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)
 
     def hyp_changed(self, hyp):
-        hyp = ' '.join(hyp).replace('[noise]', ' ').split()
-        print 'ASR:', hyp
+        #hyp = ' '.join(hyp).replace('[noise]', ' ').split()
+        words = hyp[-1].strip().split()
+        if self.current_line >= len(self.lines) - 1:
+            print "FINISHED"
+            return
+        line = self.lines[self.current_line + 1].text.split()
+        import levenstein
+        num_errors, num_ref, alignment, score = levenstein.align(line, words)
+        num_matches = 0
+        for ref_word, hyp_word in alignment:
+            if ref_word == hyp_word and ref_word != None:
+                num_matches += 1
+        score = float(num_matches) / max(len(line), len(words))
+        print 'ASR:', hyp[-1], 'REF:', self.lines[self.current_line + 1].text, 'score:', score
+        levenstein.print_alignment(alignment)
+        if score >= 0.5:
+            if self.current_line >= 0:
+                self.lines[self.current_line].highlight(False)
+            self.current_line += 1
+            self.lines[self.current_line].highlight(True)
 
     def quit(self, window):
         Gtk.main_quit()

xmlview_widgets.py

@@ -12,11 +12,6 @@ class Section(Gtk.VBox):
         self.pack_start(self.label, True, True, 5)
         self.sequences = []
 
-        #self.override_background_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1,1,1,1))
-        #self.label.override_background_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(.5,.5,.5,1))
-        #self.label.override_color(Gtk.StateFlags.NORMAL, Gdk.RGBA(1,1,1,1))
-        #self.label.override_font(Pango.FontDescription("bold 18"))
-
         num = 1
         for sequence in section.findall('./sequence'):
             self.sequences.append(Sequence(sequence, section.get('id') + '.' + str(num)))
@@ -51,6 +46,12 @@ class Line(Gtk.Label):
         self.set_halign(Gtk.Align.START)
         self.get_style_context().add_class('text-line')
     
+    def highlight(self, highlighted=True):
+        if highlighted:
+            self.get_style_context().add_class('highlighted')
+        else:
+            self.get_style_context().remove_class('highlighted')
+
 class Word:
     def __init__(self, text, start, end):
         self.text = text
@@ -68,7 +69,6 @@ class XmlView(Gtk.ScrolledWindow):
         self.add_with_viewport(self.parse_xml(filename))
 
         self.last_section = None
-        self.show_section(0)
 
     def get_view(self):
         return self
@@ -83,6 +83,9 @@ class XmlView(Gtk.ScrolledWindow):
             vbox.pack_start(self.sections[-1], True, True, 5)
         return vbox
 
-    def show_section(self, section):
-        pass
+    def get_line_iterator(self):
+        for section in self.sections:
+            for sequence in section.sequences:
+                for line in sequence.lines:
+                    yield line