first

README

+WHAT IT IS
+----------
+
+GStreamer plugin that wraps Kaldi's SingleUtteranceNnet2Decoder.
+
+HOW TO COMPILE IT
+-----------------
+
+Compile Kaldi trunk, using the shared configuration:
+In Kaldi's 'src' directory:
+
+./configure --shared
+make depend
+make
+
+Now we can compil ethis plugin:
+
+Change to 'src' of this project:
+
+cd src
+
+Compile, specifying Kaldi's root directory:
+
+KALDI_ROOT=/path/of/kaldi-trunk make
+
+This should result in 'libgstkaldionline2.so'.
+
+HOW TO USE IT
+-------------
+
+TODO

src/Makefile

+all:
+
+KALDI_ROOT:=/home/tanel/tools/kaldi-online
+
+include $(KALDI_ROOT)/src/kaldi.mk
+ifneq ($(KALDI_FLAVOR), dynamic)
+$(error Kaldi must compiled with dynamic libraries support. Run configure with --shared flag. )
+endif
+
+
+CXXFLAGS+=-I$(KALDI_ROOT)/src
+
+EXTRA_CXXFLAGS += $(shell pkg-config --cflags gstreamer-1.0)
+EXTRA_CXXFLAGS += $(shell pkg-config --cflags glib-2.0)
+
+EXTRA_LDLIBS = -pthread -lgstbase-1.0 -lgstcontroller-1.0 -lgstreamer-1.0 -lgobject-2.0 -lgmodule-2.0 -lgthread-2.0 -lrt -lglib-2.0
+
+#Kaldi shared libraries required by the GStreamer plugin
+EXTRA_LDLIBS += -lkaldi-online2 -lkaldi-lat -lkaldi-decoder -lkaldi-feat -lkaldi-transform \
+ -lkaldi-gmm -lkaldi-hmm \
+ -lkaldi-tree -lkaldi-matrix  -lkaldi-util -lkaldi-base -lkaldi-thread
+
+
+OBJFILES = gstkaldinnet2onlinedecoder.o simple-options-gst.o gst-audio-source.o kaldimarshal.o
+
+LIBNAME=gstkaldionline2
+
+LIBFILE = lib$(LIBNAME).so
+BINFILES= $(LIBFILE)
+
+all: $(LIBFILE)
+
+
+# MKL libs required when linked via shared library
+ifdef MKLROOT
+EXTRA_LDLIBS+=-lmkl_p4n -lmkl_def
+endif
+
+$(LIBFILE): $(OBJFILES)
+	$(CXX) -shared -DPIC -o $(LIBFILE) -Wl,-soname=$(LIBFILE) -Wl,--no-as-needed \
+	  -L$(KALDILIBDIR) -Wl,-rpath=$(KALDILIBDIR) $(EXTRA_LDLIBS) $(LDLIBS) $(LDFLAGS) \
+	  $(OBJFILES)
+ 
+kaldimarshal.h: kaldimarshal.list
+	glib-genmarshal --header --prefix=kaldi_marshal kaldimarshal.list > kaldimarshal.h.tmp
+	mv kaldimarshal.h.tmp kaldimarshal.h
+
+kaldimarshal.cc: kaldimarshal.list kaldimarshal.h
+	echo "#include \"glib-object.h\"" >> kaldimarshal.c.tmp
+	echo "#include \"kaldimarshal.h\"" >> kaldimarshal.c.tmp
+	glib-genmarshal --body --prefix=kaldi_marshal kaldimarshal.list >> kaldimarshal.c.tmp
+	mv kaldimarshal.c.tmp kaldimarshal.cc
+ 
+clean: 
+	-rm -f *.o *.a $(TESTFILES) $(BINFILES) kaldimarshal.h kaldimarshal.cc
+ 
+# kaldimarshal.h kaldimarshal.cc 
+depend: 
+	-$(CXX) -M $(CXXFLAGS) *.cc > .depend.mk  
+
+-include .depend.mk
+
+

src/gst-audio-source.cc

+// gst-plugin/gst-audio-source.cc
+
+// Copyright 2013  Tanel Alumae, Tallinn University of Technology
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+
+#include "gst-audio-source.h"
+
+namespace kaldi {
+
+
+GstBufferSource::GstBufferSource() :
+  ended_(false) {
+  buf_queue_ = g_async_queue_new();
+  current_buffer_ = NULL;
+  pos_in_current_buf_ = 0;
+
+  // Monophone, 16-bit input hardcoded
+  KALDI_ASSERT(sizeof(SampleType) == 2 &&
+      "The current GstBufferSource code assumes 16-bit input");
+  g_cond_init(&data_cond_);
+  g_mutex_init(&lock_);
+}
+
+GstBufferSource::~GstBufferSource() {
+  g_cond_clear(&data_cond_);
+  g_mutex_clear(&lock_);
+  g_async_queue_unref(buf_queue_);
+  if (current_buffer_) {
+    gst_buffer_unref(current_buffer_);
+    current_buffer_ = NULL;
+  }
+}
+
+void GstBufferSource::PushBuffer(GstBuffer *buf) {
+  g_mutex_lock(&lock_);
+  gst_buffer_ref(buf);
+  g_async_queue_push(buf_queue_, buf);
+  g_cond_signal(&data_cond_);
+  g_mutex_unlock(&lock_);
+}
+
+void GstBufferSource::SetEnded(bool ended) {
+  ended_ = ended;
+  g_mutex_lock(&lock_);
+  g_cond_signal(&data_cond_);
+  g_mutex_unlock(&lock_);
+}
+
+
+bool GstBufferSource::Read(Vector<BaseFloat> *data) {
+  uint32 nsamples_req = data->Dim();  // (16bit) samples requested
+  int16 buf[data->Dim()];
+  uint32 nbytes_transferred = 0;
+
+  while ((nbytes_transferred  < nsamples_req * sizeof(SampleType))) {
+    g_mutex_lock(&lock_);
+    while ((current_buffer_ == NULL) &&
+        !((g_async_queue_length(buf_queue_) == 0) && ended_)) {
+      current_buffer_ = reinterpret_cast<GstBuffer*>(g_async_queue_try_pop(buf_queue_));
+      if (current_buffer_ == NULL) {
+        g_cond_wait(&data_cond_, &lock_);
+      }
+    }
+    g_mutex_unlock(&lock_);
+    if (current_buffer_ == NULL) {
+      break;
+    }
+    uint32 nbytes_from_current =
+        std::min(nsamples_req * sizeof(SampleType) - nbytes_transferred,
+                 (gst_buffer_get_size(current_buffer_) - pos_in_current_buf_));
+    uint32 nbytes_extracted =
+        gst_buffer_extract(current_buffer_, pos_in_current_buf_,
+                           (reinterpret_cast<char *>(buf)) + nbytes_transferred,
+                           nbytes_from_current);
+    KALDI_ASSERT(nbytes_extracted == nbytes_from_current
+                 && "Unexpected number of bytes extracted from Gst buffer");
+
+    nbytes_transferred += nbytes_from_current;
+    pos_in_current_buf_ += nbytes_from_current;
+    if (pos_in_current_buf_ == gst_buffer_get_size(current_buffer_)) {
+      // we are done with the current buffer
+      gst_buffer_unref(current_buffer_);
+      current_buffer_ = NULL;
+      pos_in_current_buf_ = 0;
+    }
+  }
+
+  uint32 nsamples_received = nbytes_transferred / sizeof(SampleType);
+  for (int i = 0; i < nsamples_received ; ++i) {
+    (*data)(i) = static_cast<BaseFloat>(buf[i]);
+  }
+
+  if (nsamples_received < nsamples_req) {
+    data->Resize(nsamples_received, kCopyData);
+  }
+  return !((g_async_queue_length(buf_queue_) < sizeof(SampleType))
+      && ended_
+      && (current_buffer_ == NULL));
+}
+}

src/gst-audio-source.h

+// gst-plugin/gst-audio-source.h
+
+// Copyright 2013  Tanel Alumae, Tallinn University of Technology
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_
+#define KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_
+
+#include <matrix/kaldi-vector.h>
+#include <gst/gst.h>
+
+namespace kaldi {
+
+
+// OnlineAudioSourceItf implementation using a queue of Gst Buffers
+class GstBufferSource {
+ public:
+  typedef int16 SampleType;  // hardcoded 16-bit audio
+
+  GstBufferSource();
+
+  // Implementation of the OnlineAudioSourceItf
+  bool Read(Vector<BaseFloat> *data);
+
+  void PushBuffer(GstBuffer *buf);
+
+  void SetEnded(bool ended);
+
+  ~GstBufferSource();
+
+ private:
+
+  GAsyncQueue* buf_queue_;
+  gint pos_in_current_buf_;
+  GstBuffer *current_buffer_;
+  bool ended_;
+  GMutex lock_;
+  GCond data_cond_;
+  KALDI_DISALLOW_COPY_AND_ASSIGN(GstBufferSource);
+};
+
+}  // namespace kaldi
+
+#endif  // KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_

src/gstkaldinnet2onlinedecoder.h

+/*
+ * GStreamer
+ * Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
+ * Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
+ * Copyright (C) 2014 Tanel Alumae <<user@hostname.org>>
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Alternatively, the contents of this file may be used under the
+ * GNU Lesser General Public License Version 2.1 (the "LGPL"), in
+ * which case the following provisions apply instead of the ones
+ * mentioned above:
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __GST_KALDINNET2ONLINEDECODER_H__
+#define __GST_KALDINNET2ONLINEDECODER_H__
+
+
+#include <gst/gst.h>
+
+#include "simple-options-gst.h"
+#include "gst-audio-source.h"
+
+#include "online2/online-nnet2-decoding.h"
+#include "online2/onlinebin-util.h"
+#include "online2/online-timing.h"
+#include "online2/online-endpoint.h"
+#include "fstext/fstext-lib.h"
+#include "lat/lattice-functions.h"
+
+namespace kaldi {
+
+G_BEGIN_DECLS
+
+/* #defines don't like whitespacey bits */
+#define GST_TYPE_KALDINNET2ONLINEDECODER \
+  (gst_kaldinnet2onlinedecoder_get_type())
+#define GST_KALDINNET2ONLINEDECODER(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_KALDINNET2ONLINEDECODER,Gstkaldinnet2onlinedecoder))
+#define GST_KALDINNET2ONLINEDECODER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_KALDINNET2ONLINEDECODER,Gstkaldinnet2onlinedecoderClass))
+#define GST_IS_KALDINNET2ONLINEDECODER(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_KALDINNET2ONLINEDECODER))
+#define GST_IS_KALDINNET2ONLINEDECODER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_KALDINNET2ONLINEDECODER))
+
+typedef struct _Gstkaldinnet2onlinedecoder      Gstkaldinnet2onlinedecoder;
+typedef struct _Gstkaldinnet2onlinedecoderClass Gstkaldinnet2onlinedecoderClass;
+
+struct _Gstkaldinnet2onlinedecoder
+{
+  GstElement element;
+
+  GstPad *sinkpad, *srcpad;
+
+  gboolean silent;
+  gboolean do_endpointing;
+  GstBufferSource *audio_source;
+
+  gchar* model_rspecifier;
+  gchar* fst_rspecifier;
+  gchar* word_syms_filename;
+
+  SimpleOptionsGst *simple_options;
+  OnlineEndpointConfig *endpoint_config;
+  OnlineNnet2FeaturePipelineConfig *feature_config;
+  OnlineNnet2DecodingConfig *nnet2_decoding_config;
+
+  OnlineNnet2FeaturePipelineInfo *feature_info;
+  TransitionModel *trans_model;
+  nnet2::AmNnet *nnet;
+  fst::Fst<fst::StdArc> *decode_fst;
+  fst::SymbolTable *word_syms;
+  //OnlineNnet2FeaturePipeline *feature_pipeline;
+};
+
+struct _Gstkaldinnet2onlinedecoderClass 
+{
+  GstElementClass parent_class;
+  void (*partial_result)(GstElement *element, const gchar *result_str);
+  void (*final_result)(GstElement *element, const gchar *result_str);
+};
+
+GType gst_kaldinnet2onlinedecoder_get_type (void);
+
+G_END_DECLS
+}
+#endif /* __GST_KALDINNET2ONLINEDECODER_H__ */

src/simple-options-gst.cc

+// simple-options-gst.cc
+
+// Copyright 2014  Tanel Alumae, Tallinn University of Technology
+
+// See ../../COPYING for clarification regarding multiple authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// See the Apache 2 License for the specific language governing permissions and
+// limitations under the License.
+
+#include "simple-options-gst.h"
+#include <algorithm>
+
+namespace kaldi {
+
+void SimpleOptionsGst::Register(const std::string &name,
+                             bool *ptr,
+                             const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+
+void SimpleOptionsGst::Register(const std::string &name, int32 *ptr, const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+
+void SimpleOptionsGst::Register(const std::string &name, uint32 *ptr, const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+
+
+void SimpleOptionsGst::Register(const std::string &name, float *ptr, const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+void SimpleOptionsGst::Register(const std::string &name, double *ptr, const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+void SimpleOptionsGst::Register(const std::string &name, std::string *ptr,
+                const std::string &doc) {
+  std::string new_name = TransformName(name);
+  SimpleOptions::Register(new_name, ptr, doc);
+}
+
+
+std::string SimpleOptionsGst::TransformName(const std::string &name) {
+  std::string new_name = name;
+  std::replace( new_name.begin(), new_name.end(), '.', '-');
+  return new_name;
+}
+
+}

src/simple-options-gst.h

+#ifndef SIMPLE_OPTIONS_GST_H_
+#define SIMPLE_OPTIONS_GST_H_
+
+#include <string>
+
+#include "util/simple-options.h"
+
+namespace kaldi {
+
+// This class is the same as Kaldi's SimpleOptions except that
+// it transforms all '.' characters to '-' in options names,
+// in order to avoid GStreamer doing it itself
+class SimpleOptionsGst : public SimpleOptions {
+
+  void Register(const std::string &name, bool *ptr, const std::string &doc);
+  void Register(const std::string &name, int32 *ptr, const std::string &doc);
+  void Register(const std::string &name, uint32 *ptr, const std::string &doc);
+  void Register(const std::string &name, float *ptr, const std::string &doc);
+  void Register(const std::string &name, double *ptr, const std::string &doc);
+  void Register(const std::string &name, std::string *ptr,
+                  const std::string &doc);
+
+private:
+  std::string TransformName(const std::string &name);
+};
+
+}
+#endif  // SIMPLE_OPTIONS_GST_H_