Commit fb67515f authored by tanel's avatar tanel
Browse files

first

parents
WHAT IT IS
----------
GStreamer plugin that wraps Kaldi's SingleUtteranceNnet2Decoder.
HOW TO COMPILE IT
-----------------
Compile Kaldi trunk, using the shared configuration:
In Kaldi's 'src' directory:
./configure --shared
make depend
make
Now we can compil ethis plugin:
Change to 'src' of this project:
cd src
Compile, specifying Kaldi's root directory:
KALDI_ROOT=/path/of/kaldi-trunk make
This should result in 'libgstkaldionline2.so'.
HOW TO USE IT
-------------
TODO
all:
KALDI_ROOT:=/home/tanel/tools/kaldi-online
include $(KALDI_ROOT)/src/kaldi.mk
ifneq ($(KALDI_FLAVOR), dynamic)
$(error Kaldi must compiled with dynamic libraries support. Run configure with --shared flag. )
endif
CXXFLAGS+=-I$(KALDI_ROOT)/src
EXTRA_CXXFLAGS += $(shell pkg-config --cflags gstreamer-1.0)
EXTRA_CXXFLAGS += $(shell pkg-config --cflags glib-2.0)
EXTRA_LDLIBS = -pthread -lgstbase-1.0 -lgstcontroller-1.0 -lgstreamer-1.0 -lgobject-2.0 -lgmodule-2.0 -lgthread-2.0 -lrt -lglib-2.0
#Kaldi shared libraries required by the GStreamer plugin
EXTRA_LDLIBS += -lkaldi-online2 -lkaldi-lat -lkaldi-decoder -lkaldi-feat -lkaldi-transform \
-lkaldi-gmm -lkaldi-hmm \
-lkaldi-tree -lkaldi-matrix -lkaldi-util -lkaldi-base -lkaldi-thread
OBJFILES = gstkaldinnet2onlinedecoder.o simple-options-gst.o gst-audio-source.o kaldimarshal.o
LIBNAME=gstkaldionline2
LIBFILE = lib$(LIBNAME).so
BINFILES= $(LIBFILE)
all: $(LIBFILE)
# MKL libs required when linked via shared library
ifdef MKLROOT
EXTRA_LDLIBS+=-lmkl_p4n -lmkl_def
endif
$(LIBFILE): $(OBJFILES)
$(CXX) -shared -DPIC -o $(LIBFILE) -Wl,-soname=$(LIBFILE) -Wl,--no-as-needed \
-L$(KALDILIBDIR) -Wl,-rpath=$(KALDILIBDIR) $(EXTRA_LDLIBS) $(LDLIBS) $(LDFLAGS) \
$(OBJFILES)
kaldimarshal.h: kaldimarshal.list
glib-genmarshal --header --prefix=kaldi_marshal kaldimarshal.list > kaldimarshal.h.tmp
mv kaldimarshal.h.tmp kaldimarshal.h
kaldimarshal.cc: kaldimarshal.list kaldimarshal.h
echo "#include \"glib-object.h\"" >> kaldimarshal.c.tmp
echo "#include \"kaldimarshal.h\"" >> kaldimarshal.c.tmp
glib-genmarshal --body --prefix=kaldi_marshal kaldimarshal.list >> kaldimarshal.c.tmp
mv kaldimarshal.c.tmp kaldimarshal.cc
clean:
-rm -f *.o *.a $(TESTFILES) $(BINFILES) kaldimarshal.h kaldimarshal.cc
# kaldimarshal.h kaldimarshal.cc
depend:
-$(CXX) -M $(CXXFLAGS) *.cc > .depend.mk
-include .depend.mk
// gst-plugin/gst-audio-source.cc
// Copyright 2013 Tanel Alumae, Tallinn University of Technology
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include "gst-audio-source.h"
namespace kaldi {
GstBufferSource::GstBufferSource() :
ended_(false) {
buf_queue_ = g_async_queue_new();
current_buffer_ = NULL;
pos_in_current_buf_ = 0;
// Monophone, 16-bit input hardcoded
KALDI_ASSERT(sizeof(SampleType) == 2 &&
"The current GstBufferSource code assumes 16-bit input");
g_cond_init(&data_cond_);
g_mutex_init(&lock_);
}
GstBufferSource::~GstBufferSource() {
g_cond_clear(&data_cond_);
g_mutex_clear(&lock_);
g_async_queue_unref(buf_queue_);
if (current_buffer_) {
gst_buffer_unref(current_buffer_);
current_buffer_ = NULL;
}
}
void GstBufferSource::PushBuffer(GstBuffer *buf) {
g_mutex_lock(&lock_);
gst_buffer_ref(buf);
g_async_queue_push(buf_queue_, buf);
g_cond_signal(&data_cond_);
g_mutex_unlock(&lock_);
}
void GstBufferSource::SetEnded(bool ended) {
ended_ = ended;
g_mutex_lock(&lock_);
g_cond_signal(&data_cond_);
g_mutex_unlock(&lock_);
}
bool GstBufferSource::Read(Vector<BaseFloat> *data) {
uint32 nsamples_req = data->Dim(); // (16bit) samples requested
int16 buf[data->Dim()];
uint32 nbytes_transferred = 0;
while ((nbytes_transferred < nsamples_req * sizeof(SampleType))) {
g_mutex_lock(&lock_);
while ((current_buffer_ == NULL) &&
!((g_async_queue_length(buf_queue_) == 0) && ended_)) {
current_buffer_ = reinterpret_cast<GstBuffer*>(g_async_queue_try_pop(buf_queue_));
if (current_buffer_ == NULL) {
g_cond_wait(&data_cond_, &lock_);
}
}
g_mutex_unlock(&lock_);
if (current_buffer_ == NULL) {
break;
}
uint32 nbytes_from_current =
std::min(nsamples_req * sizeof(SampleType) - nbytes_transferred,
(gst_buffer_get_size(current_buffer_) - pos_in_current_buf_));
uint32 nbytes_extracted =
gst_buffer_extract(current_buffer_, pos_in_current_buf_,
(reinterpret_cast<char *>(buf)) + nbytes_transferred,
nbytes_from_current);
KALDI_ASSERT(nbytes_extracted == nbytes_from_current
&& "Unexpected number of bytes extracted from Gst buffer");
nbytes_transferred += nbytes_from_current;
pos_in_current_buf_ += nbytes_from_current;
if (pos_in_current_buf_ == gst_buffer_get_size(current_buffer_)) {
// we are done with the current buffer
gst_buffer_unref(current_buffer_);
current_buffer_ = NULL;
pos_in_current_buf_ = 0;
}
}
uint32 nsamples_received = nbytes_transferred / sizeof(SampleType);
for (int i = 0; i < nsamples_received ; ++i) {
(*data)(i) = static_cast<BaseFloat>(buf[i]);
}
if (nsamples_received < nsamples_req) {
data->Resize(nsamples_received, kCopyData);
}
return !((g_async_queue_length(buf_queue_) < sizeof(SampleType))
&& ended_
&& (current_buffer_ == NULL));
}
}
// gst-plugin/gst-audio-source.h
// Copyright 2013 Tanel Alumae, Tallinn University of Technology
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#ifndef KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_
#define KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_
#include <matrix/kaldi-vector.h>
#include <gst/gst.h>
namespace kaldi {
// OnlineAudioSourceItf implementation using a queue of Gst Buffers
class GstBufferSource {
public:
typedef int16 SampleType; // hardcoded 16-bit audio
GstBufferSource();
// Implementation of the OnlineAudioSourceItf
bool Read(Vector<BaseFloat> *data);
void PushBuffer(GstBuffer *buf);
void SetEnded(bool ended);
~GstBufferSource();
private:
GAsyncQueue* buf_queue_;
gint pos_in_current_buf_;
GstBuffer *current_buffer_;
bool ended_;
GMutex lock_;
GCond data_cond_;
KALDI_DISALLOW_COPY_AND_ASSIGN(GstBufferSource);
};
} // namespace kaldi
#endif // KALDI_GST_PLUGIN_GST_AUDIO_SOURCE_H_
This diff is collapsed.
/*
* GStreamer
* Copyright (C) 2005 Thomas Vander Stichele <thomas@apestaart.org>
* Copyright (C) 2005 Ronald S. Bultje <rbultje@ronald.bitfreak.net>
* Copyright (C) 2014 Tanel Alumae <<user@hostname.org>>
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*
* Alternatively, the contents of this file may be used under the
* GNU Lesser General Public License Version 2.1 (the "LGPL"), in
* which case the following provisions apply instead of the ones
* mentioned above:
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#ifndef __GST_KALDINNET2ONLINEDECODER_H__
#define __GST_KALDINNET2ONLINEDECODER_H__
#include <gst/gst.h>
#include "simple-options-gst.h"
#include "gst-audio-source.h"
#include "online2/online-nnet2-decoding.h"
#include "online2/onlinebin-util.h"
#include "online2/online-timing.h"
#include "online2/online-endpoint.h"
#include "fstext/fstext-lib.h"
#include "lat/lattice-functions.h"
namespace kaldi {
G_BEGIN_DECLS
/* #defines don't like whitespacey bits */
#define GST_TYPE_KALDINNET2ONLINEDECODER \
(gst_kaldinnet2onlinedecoder_get_type())
#define GST_KALDINNET2ONLINEDECODER(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_KALDINNET2ONLINEDECODER,Gstkaldinnet2onlinedecoder))
#define GST_KALDINNET2ONLINEDECODER_CLASS(klass) \
(G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_KALDINNET2ONLINEDECODER,Gstkaldinnet2onlinedecoderClass))
#define GST_IS_KALDINNET2ONLINEDECODER(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_KALDINNET2ONLINEDECODER))
#define GST_IS_KALDINNET2ONLINEDECODER_CLASS(klass) \
(G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_KALDINNET2ONLINEDECODER))
typedef struct _Gstkaldinnet2onlinedecoder Gstkaldinnet2onlinedecoder;
typedef struct _Gstkaldinnet2onlinedecoderClass Gstkaldinnet2onlinedecoderClass;
struct _Gstkaldinnet2onlinedecoder
{
GstElement element;
GstPad *sinkpad, *srcpad;
gboolean silent;
gboolean do_endpointing;
GstBufferSource *audio_source;
gchar* model_rspecifier;
gchar* fst_rspecifier;
gchar* word_syms_filename;
SimpleOptionsGst *simple_options;
OnlineEndpointConfig *endpoint_config;
OnlineNnet2FeaturePipelineConfig *feature_config;
OnlineNnet2DecodingConfig *nnet2_decoding_config;
OnlineNnet2FeaturePipelineInfo *feature_info;
TransitionModel *trans_model;
nnet2::AmNnet *nnet;
fst::Fst<fst::StdArc> *decode_fst;
fst::SymbolTable *word_syms;
//OnlineNnet2FeaturePipeline *feature_pipeline;
};
struct _Gstkaldinnet2onlinedecoderClass
{
GstElementClass parent_class;
void (*partial_result)(GstElement *element, const gchar *result_str);
void (*final_result)(GstElement *element, const gchar *result_str);
};
GType gst_kaldinnet2onlinedecoder_get_type (void);
G_END_DECLS
}
#endif /* __GST_KALDINNET2ONLINEDECODER_H__ */
// simple-options-gst.cc
// Copyright 2014 Tanel Alumae, Tallinn University of Technology
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#include "simple-options-gst.h"
#include <algorithm>
namespace kaldi {
void SimpleOptionsGst::Register(const std::string &name,
bool *ptr,
const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
void SimpleOptionsGst::Register(const std::string &name, int32 *ptr, const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
void SimpleOptionsGst::Register(const std::string &name, uint32 *ptr, const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
void SimpleOptionsGst::Register(const std::string &name, float *ptr, const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
void SimpleOptionsGst::Register(const std::string &name, double *ptr, const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
void SimpleOptionsGst::Register(const std::string &name, std::string *ptr,
const std::string &doc) {
std::string new_name = TransformName(name);
SimpleOptions::Register(new_name, ptr, doc);
}
std::string SimpleOptionsGst::TransformName(const std::string &name) {
std::string new_name = name;
std::replace( new_name.begin(), new_name.end(), '.', '-');
return new_name;
}
}
#ifndef SIMPLE_OPTIONS_GST_H_
#define SIMPLE_OPTIONS_GST_H_
#include <string>
#include "util/simple-options.h"
namespace kaldi {
// This class is the same as Kaldi's SimpleOptions except that
// it transforms all '.' characters to '-' in options names,
// in order to avoid GStreamer doing it itself
class SimpleOptionsGst : public SimpleOptions {
void Register(const std::string &name, bool *ptr, const std::string &doc);
void Register(const std::string &name, int32 *ptr, const std::string &doc);
void Register(const std::string &name, uint32 *ptr, const std::string &doc);
void Register(const std::string &name, float *ptr, const std::string &doc);
void Register(const std::string &name, double *ptr, const std::string &doc);
void Register(const std::string &name, std::string *ptr,
const std::string &doc);
private:
std::string TransformName(const std::string &name);
};
}
#endif // SIMPLE_OPTIONS_GST_H_
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment